Refactor normalization workflow

Author: IvanildoBarauna

README.md

@@ -78,6 +78,47 @@ df = client.api_to_dataframe(data)
 print(df)
 ```
 
+### Normalization options
+
+`ClientBuilder.api_to_dataframe` accepts keyword arguments that are
+forwarded to [`pandas.json_normalize`](https://pandas.pydata.org/docs/reference/api/pandas.json_normalize.html).
+This makes it possible to flatten nested payloads and include metadata
+fields easily:
+
+```python
+payload = {
+    "meta": {"page": 1},
+    "items": [
+        {"id": 1, "value": "alpha"},
+        {"id": 2, "value": "beta"},
+    ],
+}
+
+client = ClientBuilder(endpoint="https://api.example.com")
+
+df = client.api_to_dataframe(
+    payload,
+    record_path="items",
+    meta=[["meta", "page"]],
+    errors="ignore",  # optionally avoid raising when paths are missing
+)
+```
+
+When complex transformations are required before normalization, provide a
+`transformer` callable to the constructor. The callable receives the JSON
+payload returned by `get_api_data` and can return any structure supported
+by `pandas.json_normalize`:
+
+```python
+client = ClientBuilder(
+    endpoint="https://api.example.com",
+    transformer=lambda payload: payload["data"],
+)
+
+data = client.get_api_data()
+df = client.api_to_dataframe(data)
+```
+
 ## Important notes:
 * **Opcionals Parameters:** The params timeout, retry_strategy and headers are opcionals.
 * **Default Params Value:** By default the quantity of retries is 3 and the time between retries is 1 second, but you can define manually.

src/api_to_dataframe/controller/client_builder.py

@@ -1,3 +1,5 @@
+from typing import Any, Callable, Optional, Sequence, Union
+
 from api_to_dataframe.models.retainer import retry_strategies, Strategies
 from api_to_dataframe.models.get_data import GetData
 from api_to_dataframe.utils.logger import logger
@@ -12,6 +14,7 @@ def __init__(  # pylint: disable=too-many-positional-arguments,too-many-argument
         retries: int = 3,
         initial_delay: int = 1,
         connection_timeout: int = 1,
+        transformer: Optional[Callable[[Any], Any]] = None,
     ):
         """
         Initializes the ClientBuilder object.
@@ -23,6 +26,8 @@ def __init__(  # pylint: disable=too-many-positional-arguments,too-many-argument
             retries (int): The number of times to retry a failed request. Defaults to 3.
             initial_delay (int): The delay between retries in seconds. Defaults to 1.
             connection_timeout (int): The timeout for the connection in seconds. Defaults to 1.
+            transformer (Optional[Callable[[Any], Any]]): Optional callable to
+                transform the JSON payload before DataFrame normalization.
 
         Raises:
             ValueError: If endpoint is an empty string.
@@ -56,6 +61,7 @@ def __init__(  # pylint: disable=too-many-positional-arguments,too-many-argument
         self.headers = headers
         self.retries = retries
         self.delay = initial_delay
+        self.transformer = transformer
 
     @retry_strategies
     def get_api_data(self):
@@ -77,19 +83,39 @@ def get_api_data(self):
 
         return response.json()
 
-    @staticmethod
-    def api_to_dataframe(response: dict):
-        """
-        Converts an API response to a DataFrame.
-
-        This function takes a dictionary response from an API,
-        uses the `to_dataframe` function from the `GetData` class
-        to convert it into a DataFrame, and logs the operation as successful.
+    def api_to_dataframe(
+        self,
+        response: Any,
+        *,
+        record_path: Optional[Union[str, Sequence[str]]] = None,
+        meta: Optional[Sequence[Union[str, Sequence[str]]]] = None,
+        errors: str = "raise",
+        max_level: Optional[int] = None,
+    ):
+        """Normalize an API response into a pandas DataFrame.
 
         Args:
-            response (dict): The dictionary containing the API response.
+            response (Any): The already decoded API response payload.
+            record_path (Optional[Union[str, Sequence[str]]]): Path to nested
+                records passed to :func:`pandas.json_normalize`.
+            meta (Optional[Sequence[Union[str, Sequence[str]]]]): Metadata keys to
+                include as columns in the result.
+            errors (str): Error handling strategy to forward to
+                :func:`pandas.json_normalize` ("raise" or "ignore").
+            max_level (Optional[int]): Maximum depth for record flattening.
 
         Returns:
-            DataFrame: A pandas DataFrame containing the data from the API response.
+            pandas.DataFrame: A DataFrame containing the normalized payload.
         """
-        return GetData.to_dataframe(response)
+
+        data = response
+        if self.transformer is not None:
+            data = self.transformer(response)
+
+        return GetData.to_dataframe(
+            data,
+            record_path=record_path,
+            meta=meta,
+            errors=errors,
+            max_level=max_level,
+        )

src/api_to_dataframe/models/get_data.py

@@ -1,5 +1,8 @@
-import requests
+from typing import Any, Optional, Sequence, Union
+
 import pandas as pd
+import requests
+
 from api_to_dataframe.utils.logger import logger
 
 
@@ -15,13 +18,54 @@ def get_response(endpoint: str, headers: dict, connection_timeout: int):
         return response
 
     @staticmethod
-    def to_dataframe(response):
-        df = pd.DataFrame(response)
+    def to_dataframe(
+        response: Any,
+        *,
+        record_path: Optional[Union[str, Sequence[str]]] = None,
+        meta: Optional[Sequence[Union[str, Sequence[str]]]] = None,
+        errors: str = "raise",
+        max_level: Optional[int] = None,
+    ) -> pd.DataFrame:
+        """Convert an API response object into a pandas DataFrame.
+
+        Args:
+            response (Any): The API response already decoded to a Python object.
+            record_path (Optional[Union[str, Sequence[str]]]): The path to records for
+                nested data structures accepted by :func:`pandas.json_normalize`.
+            meta (Optional[Sequence[Union[str, Sequence[str]]]]): Additional metadata
+                to include as columns in the resulting DataFrame.
+            errors (str): Error handling strategy from
+                :func:`pandas.json_normalize` ("raise" or "ignore").
+            max_level (Optional[int]): Max depth to normalize nested records.
+
+        Returns:
+            pandas.DataFrame: A DataFrame containing the normalized data.
+
+        Raises:
+            ValueError: If the normalization results in an empty DataFrame while the
+                error strategy is not set to ``"ignore"``.
+        """
+
+        if response is None or response == "":
+            error_msg = "::: Response payload is empty :::"
+            logger.error(error_msg)
+            raise ValueError(error_msg)
+
+        try:
+            dataframe = pd.json_normalize(
+                response,
+                record_path=record_path,
+                meta=meta,
+                errors=errors,
+                max_level=max_level,
+            )
+        except Exception as exc:  # pragma: no cover - defensive logging
+            logger.error("::: Failed to normalize response: %s :::", exc)
+            raise
 
-        # Check if DataFrame is empty
-        if df.empty:
+        if dataframe.empty and errors != "ignore":
             error_msg = "::: DataFrame is empty :::"
             logger.error(error_msg)
             raise ValueError(error_msg)
 
-        return df
+        return dataframe

tests/test_controller_client_builder.py

@@ -1,12 +1,16 @@
-import pytest
+"""Tests for the ClientBuilder controller."""
+
 import pandas as pd
+import pytest
 import responses
 
 from api_to_dataframe import ClientBuilder, RetryStrategies
 
 
 @pytest.fixture()
 def client_setup():
+    """Provide a ClientBuilder instance for constructor related tests."""
+
     new_client = ClientBuilder(
         endpoint="https://economia.awesomeapi.com.br/last/USD-BRL"
     )
@@ -15,13 +19,17 @@ def client_setup():
 
 @pytest.fixture()
 def response_setup():
-    new_client = ClientBuilder(
-        endpoint="https://economia.awesomeapi.com.br/last/USD-BRL"
-    )
-    return new_client.get_api_data()
+    """Provide a simple payload used to build DataFrames."""
+
+    return [
+        {"code": "USD", "name": "Dollar"},
+        {"code": "BRL", "name": "Real"},
+    ]
 
 
 def test_constructor_raises():
+    """Ensure validation errors are raised for invalid constructor arguments."""
+
     with pytest.raises(ValueError):
         ClientBuilder(endpoint="")
 
@@ -59,6 +67,8 @@ def test_constructor_raises():
 
 
 def test_constructor_with_param(client_setup):  # pylint: disable=redefined-outer-name
+    """Ensure the endpoint argument is stored on the instance."""
+
     expected_result = "https://economia.awesomeapi.com.br/last/USD-BRL"
     new_client = client_setup
     assert new_client.endpoint == expected_result
@@ -87,14 +97,29 @@ def test_constructor_with_retry_strategy():
     assert client.delay == 2
 
 
+@responses.activate
 def test_response_to_json(client_setup):  # pylint: disable=redefined-outer-name
+    """Ensure the API response is decoded to JSON."""
+
     new_client = client_setup
+    endpoint = new_client.endpoint
+
+    responses.add(
+        responses.GET,
+        endpoint,
+        json={"status": "ok"},
+        status=200,
+    )
+
     response = new_client.get_api_data()  # pylint: disable=protected-access
     assert isinstance(response, dict)
 
 
 def test_to_dataframe(response_setup):  # pylint: disable=redefined-outer-name
-    df = ClientBuilder.api_to_dataframe(response_setup)
+    """Ensure responses are converted into DataFrames using instance method."""
+
+    client = ClientBuilder(endpoint="https://economia.awesomeapi.com.br/last/USD-BRL")
+    df = client.api_to_dataframe(response_setup)
     assert isinstance(df, pd.DataFrame)
 
 
@@ -118,3 +143,26 @@ def test_get_api_data_with_mocked_response():
     assert response == expected_data
     assert len(responses.calls) == 1
     assert responses.calls[0].request.url == endpoint
+
+
+def test_api_to_dataframe_supports_custom_normalization():
+    """Ensure normalization parameters are forwarded to the GetData helper."""
+
+    payload = {
+        "meta": {"page": 1},
+        "items": [
+            {"id": 1, "name": "First"},
+            {"id": 2, "name": "Second"},
+        ],
+    }
+
+    client = ClientBuilder(endpoint="https://economia.awesomeapi.com.br/last/USD-BRL")
+
+    dataframe = client.api_to_dataframe(
+        payload,
+        record_path="items",
+        meta=[["meta", "page"]],
+    )
+
+    assert list(dataframe.columns) == ["id", "name", "meta.page"]
+    assert dataframe.iloc[0]["meta.page"] == 1