Add request handling module (#206)

* Crete base DCResponse

DCResponse just wraps around the api response

* NodeResponse

* ObservationResponse

* ResolveResponse

* SparqlResponse

* fix return

* consistent naming

* Update response.py

* tests for response utilities

* Move response scripts and tests

* Improve endpoint docstrings

* Implement custom errors

* Add tests for custom errors
Move file to match history

* isort

isort imports with google profile

* move tests to right folder

* correct module structure (imports)

* isort tests

* Fix name typo

* Move response scripts and tests

* Model node response

Adds:
- A Node dataclass which models an individual node (with dcid, name, etc)
- A NodeGroup dataclass which represents a list of Node objects
- An Arcs dataclass, which represents arcs (with a label and containing NodeGroups)
- A Properties dataclass which represents a list of Properties

* Model observation response

Adds:
- An Observation dataclass which models an individual observation (date, value)
- An OrderedFacets dataclass which models the 'ordered facets' of observations
- A Variable dataclass which represents a variable data (grouped by entity)
- A Facet class which represents the metadata for a facet

* Model resolve response

Adds:
- A candidate dataclass to model candidates in the response (with dcid and dominanType)
- An Entity dataclass to model entities with their resolution candidates

* Model sparql response

Adds:
- A Cell dataclass to model a single cell in a row
- A Row modelling a row with cells.

* resolve conflicts (rebase)

Adds tests for node, observation, resolve and sparql models.

* Add tests for custom errors

* isort

isort imports with google profile

* Create request_handling.py

Validates instance.
Builds header.
Manages post requests, including multi-page responses

* Create test_request_handling.py

* Update docstring

Fixes a typo on the return type of the post_request function

Author: jm-rivera

…lient/tests/utils/test_error_handling.py …t/tests/endpoints/test_error_handling.pydatacommons_client/tests/utils/test_error_handling.py renamed to datacommons_client/tests/endpoints/test_error_handling.py datacommons_client/tests/utils/test_error_handling.py renamed to datacommons_client/tests/endpoints/test_error_handling.py

@@ -1,12 +1,12 @@
 from requests import Request
 from requests import Response
 
-from datacommons_client.utils.error_handling import APIError
-from datacommons_client.utils.error_handling import DataCommonsError
-from datacommons_client.utils.error_handling import DCAuthenticationError
-from datacommons_client.utils.error_handling import DCConnectionError
-from datacommons_client.utils.error_handling import DCStatusError
-from datacommons_client.utils.error_handling import InvalidDCInstanceError
+from datacommons_client.utils.error_hanlding import APIError
+from datacommons_client.utils.error_hanlding import DataCommonsError
+from datacommons_client.utils.error_hanlding import DCAuthenticationError
+from datacommons_client.utils.error_hanlding import DCConnectionError
+from datacommons_client.utils.error_hanlding import DCStatusError
+from datacommons_client.utils.error_hanlding import InvalidDCInstanceError
 
 
 def test_data_commons_error_default_message():

datacommons_client/tests/endpoints/test_request_handling.py

@@ -0,0 +1,305 @@
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import pytest
+import requests
+
+from datacommons_client.utils.error_hanlding import (
+    DCAuthenticationError,
+    DCConnectionError,
+    DCStatusError,
+    APIError,
+)
+from datacommons_client.utils.error_hanlding import InvalidDCInstanceError
+from datacommons_client.utils.request_handling import _check_instance_is_valid
+from datacommons_client.utils.request_handling import _fetch_with_pagination
+from datacommons_client.utils.request_handling import _merge_values
+from datacommons_client.utils.request_handling import _recursively_merge_dicts
+from datacommons_client.utils.request_handling import _send_post_request
+from datacommons_client.utils.request_handling import build_headers
+from datacommons_client.utils.request_handling import post_request
+from datacommons_client.utils.request_handling import resolve_instance_url
+
+
+def test_resolve_instance_url_default():
+    """Tests resolving the default Data Commons instance."""
+    assert (
+        resolve_instance_url("datacommons.org")
+        == "https://api.datacommons.org/v2"
+    )
+
+
+@patch("requests.get")
+def test_check_instance_is_valid_request_exception(mock_get):
+    """Tests that a RequestException raises InvalidDCInstanceError."""
+    mock_get.side_effect = requests.exceptions.RequestException(
+        "Request failed"
+    )
+    with pytest.raises(InvalidDCInstanceError):
+        _check_instance_is_valid("https://invalid-instance")
+
+
+@patch("requests.post")
+def test_send_post_request_connection_error(mock_post):
+    """Tests that a ConnectionError raises DCConnectionError."""
+    mock_post.side_effect = requests.exceptions.ConnectionError(
+        "Connection failed"
+    )
+    with pytest.raises(DCConnectionError):
+        _send_post_request("https://api.test.com", {}, {})
+
+
+@patch("datacommons_client.utils.request_handling._check_instance_is_valid")
+def test_resolve_instance_url_custom(mock_check_instance_is_valid):
+    """Tests resolving a custom Data Commons instance."""
+    mock_check_instance_is_valid.return_value = (
+        "https://custom-instance/core/api/v2"
+    )
+
+    assert (
+        resolve_instance_url("custom-instance")
+        == "https://custom-instance/core/api/v2"
+    )
+    mock_check_instance_is_valid.assert_called_once_with(
+        "https://custom-instance/core/api/v2"
+    )
+
+
+@patch("requests.get")
+def test_check_instance_is_valid_valid(mock_get):
+    """Tests that a valid instance URL is correctly validated."""
+
+    # Create a mock response object with the expected JSON data and status code
+    mock_response = MagicMock()
+    mock_response.json.return_value = {"data": {"country/GTM": {}}}
+    mock_response.status_code = 200
+    mock_get.return_value = mock_response
+
+    # Mock the instance URL to test
+    instance_url = "https://valid-instance"
+
+    # Assert that the instance URL is returned if it is valid
+    assert _check_instance_is_valid(instance_url) == instance_url
+    mock_get.assert_called_once_with(
+        f"{instance_url}/node?nodes=country%2FGTM&property=->name"
+    )
+
+
+@patch("requests.get")
+def test_check_instance_is_valid_invalid(mock_get):
+    """Tests that an invalid instance URL raises the appropriate exception."""
+    mock_response = MagicMock()
+    mock_response.json.return_value = {"error": "Not Found"}
+    mock_response.status_code = 404
+    mock_get.return_value = mock_response
+
+    with pytest.raises(InvalidDCInstanceError):
+        _check_instance_is_valid("https://invalid-instance")
+
+
+@patch("requests.post")
+def test_send_post_request_500_status_error(mock_post):
+    """Tests that a 500-level HTTP error raises DCStatusError."""
+    mock_response = MagicMock()
+    mock_response.status_code = 500
+    mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError(
+        response=mock_response
+    )
+    mock_post.return_value = mock_response
+
+    with pytest.raises(DCStatusError):
+        _send_post_request("https://api.test.com", {}, {})
+
+
+@patch("requests.post")
+def test_send_post_request_other_http_error(mock_post):
+    """Tests that non-500 HTTP errors raise APIError."""
+    mock_response = MagicMock()
+    mock_response.status_code = 404
+    mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError(
+        response=mock_response
+    )
+    mock_post.return_value = mock_response
+
+    with pytest.raises(APIError):
+        _send_post_request("https://api.test.com", {}, {})
+
+
+def test_build_headers_with_api_key():
+    """Tests building headers with an API key."""
+    headers = build_headers("test-api-key")
+    assert headers["Content-Type"] == "application/json"
+    assert headers["X-API-Key"] == "test-api-key"
+
+
+def test_build_headers_without_api_key():
+    """Tests building headers without an API key."""
+    headers = build_headers()
+    assert headers["Content-Type"] == "application/json"
+    assert "X-API-Key" not in headers
+
+
+@patch("requests.post")
+def test_send_post_request_success(mock_post):
+    """Tests a successful POST request."""
+
+    # Create a mock response object with the expected JSON data and status code
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.json.return_value = {"success": True}
+    mock_post.return_value = mock_response
+
+    # Mock the POST request
+    url = "https://api.test.com"
+    payload = {"key": "value"}
+    headers = {"Content-Type": "application/json"}
+
+    response = _send_post_request(url, payload, headers)
+    assert response.status_code == 200
+    assert response.json() == {"success": True}
+
+
+@patch("requests.post")
+def test_send_post_request_http_error(mock_post):
+    """Tests handling an HTTP error during a POST request."""
+    # Create a mock response object with a 401 status code
+    mock_response = MagicMock()
+    mock_response.status_code = 401
+    mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError(
+        response=mock_response
+    )
+    mock_post.return_value = mock_response
+
+    # Mock the POST request and assert that a DCAuthenticationError is raised
+    with pytest.raises(DCAuthenticationError):
+        _send_post_request("https://api.test.com", {}, {})
+
+
+def test_recursively_merge_dicts():
+    """Tests recursive merging of dictionaries."""
+    # Test merging two dictionaries with nested dictionaries and lists
+    base = {"a": {"b": 1}, "c": [1, 2]}
+    new = {"a": {"d": 2}, "c": [3], "e": 5}
+    result = _recursively_merge_dicts(base, new)
+
+    # Assert that the dictionaries are merged correctly
+    assert result == {"a": {"b": 1, "d": 2}, "c": [1, 2, 3], "e": 5}
+
+
+def test_merge_values_dicts():
+    """Tests merging of dictionary values."""
+    base = {"a": 1}
+    new = {"b": 2}
+    result = _merge_values(base, new)
+    assert result == {"a": 1, "b": 2}
+
+
+def test_merge_values_lists():
+    """Tests merging of list values."""
+    base = [1, 2]
+    new = [3, 4]
+    result = _merge_values(base, new)
+    assert result == [1, 2, 3, 4]
+
+
+def test_merge_values_other():
+    """Tests merging non-dict, non-list values."""
+    base = "value1"
+    new = "value2"
+    result = _merge_values(base, new)
+    assert result == ["value1", "value2"]
+
+
+def test_merge_values_complex_conflict():
+    """Tests merging deeply nested, repeated objects."""
+
+    # Nested but simple base
+    base = {
+        "key1": {
+            "nested1": {
+                "subkey1": "value1",
+                "subkey2": [1, 2],
+            },
+            "nested2": "value2",
+        },
+        "key2": [1, 2, 3],
+        "key3": "conflict1",
+    }
+
+    # Nested but complex new
+    new = {
+        "key1": {
+            "nested1": {
+                "subkey1": "new_value1",  # Already in base
+                "subkey3": "new_value2",  # New key
+            },
+            "nested2": ["new_value3"],  # Conflicts with base (type change)
+        },
+        "key2": [4, 5],  # Should merge lists
+        "key3": "conflict2",  # Should create a list due to conflict
+        "key4": {"new_nested": "new_value4"},  # New key
+    }
+
+    # Expected result which keeps all data
+    expected_result = {
+        "key1": {
+            "nested1": {
+                "subkey1": ["value1", "new_value1"],
+                "subkey2": [1, 2],
+                "subkey3": "new_value2",
+            },
+            "nested2": ["value2", ["new_value3"]],
+        },
+        "key2": [1, 2, 3, 4, 5],
+        "key3": ["conflict1", "conflict2"],
+        "key4": {"new_nested": "new_value4"},
+    }
+
+    result = _merge_values(base, new)
+    assert result == expected_result
+
+
+@patch("datacommons_client.utils.request_handling._send_post_request")
+def test_fetch_with_pagination(mock_send_post_request):
+    """Tests fetching and merging paginated API responses."""
+
+    # Mock the response JSON data for two pages
+    mock_response = MagicMock()
+    mock_response.json.side_effect = [
+        {"data": {"page1": True}, "nextToken": "token1"},
+        {"data": {"page2": True}},
+    ]
+    mock_send_post_request.side_effect = [mock_response, mock_response]
+
+    # Mock the POST request and assert that the results are merged correctly
+    url = "https://api.test.com"
+    payload = {}
+    headers = {}
+
+    result = _fetch_with_pagination(url, payload, headers)
+    assert result == {"data": {"page1": True, "page2": True}}
+
+
+@patch("datacommons_client.utils.request_handling._send_post_request")
+def test_fetch_with_pagination_invalid_json(mock_send_post_request):
+    """Tests that invalid JSON response raises APIError."""
+    mock_response = MagicMock()
+    mock_response.json.side_effect = ValueError("Invalid JSON")
+    mock_send_post_request.return_value = mock_response
+
+    with pytest.raises(APIError):
+        _fetch_with_pagination("https://api.test.com", {}, {})
+
+
+@patch("datacommons_client.utils.request_handling._fetch_with_pagination")
+def test_post_request(mock_fetch_with_pagination):
+    """Tests the `post_request` function with mock pagination."""
+    mock_fetch_with_pagination.return_value = {"result": "data"}
+    result = post_request("https://api.test.com", {}, {})
+    assert result == {"result": "data"}
+
+
+def test_post_request_invalid_payload():
+    """Tests that a non-dictionary payload raises a ValueError."""
+    with pytest.raises(ValueError):
+        post_request("https://api.test.com", ["not", "a", "dict"], {})

datacommons_client/tests/endpoints/test_response.py

@@ -1,15 +1,15 @@
-from datacommons_client.endpoints.response import _unpack_arcs
-from datacommons_client.endpoints.response import DCResponse
-from datacommons_client.endpoints.response import extract_observations
-from datacommons_client.endpoints.response import flatten_properties
-from datacommons_client.endpoints.response import NodeResponse
-from datacommons_client.endpoints.response import ObservationResponse
-from datacommons_client.endpoints.response import ResolveResponse
-from datacommons_client.endpoints.response import SparqlResponse
 from datacommons_client.models.observation import Facet
 from datacommons_client.models.observation import Observation
 from datacommons_client.models.observation import OrderedFacets
 from datacommons_client.models.observation import Variable
+from datacommons_client.utils.response import _unpack_arcs
+from datacommons_client.utils.response import DCResponse
+from datacommons_client.utils.response import extract_observations
+from datacommons_client.utils.response import flatten_properties
+from datacommons_client.utils.response import NodeResponse
+from datacommons_client.utils.response import ObservationResponse
+from datacommons_client.utils.response import ResolveResponse
+from datacommons_client.utils.response import SparqlResponse
 
 ### ----- Test DCResponse ----- ###