Merge pull request #86 from ParclLabs/autopaginate

bump property_v2 limit + autopaginate

Author: bhagyasharma

CHANGELOG.md

@@ -1,6 +1,12 @@
+### v1.14.2
+- Add autopaginate support for `property_v2.search` endpoint, increase default limit to 100,000.
+
 ### v1.14.1
 - Update `property_v2.search` endpoint params.
 
+### v1.14.0
+- Simplification of parameter validation, reduce complexity of code.
+
 ### v1.13.0
 - Add support for `property_v2.search` endpoint.
 

README.md

@@ -482,11 +482,13 @@ addresses = client.property_address.search.retrieve(
 
 ### Property Search V2 <a id="property-search-v2"></a>
 
-Gets a list of unique properties and their associated metadata and events based on a set of filters. Use one of three search methods:
+Gets a list of unique properties and their associated metadata and events based on a set of property, event, and owner filters. Use one of three search methods:
 1. `parcl_ids`
 2. `parcl_property_ids`
 3. `geo_coordinates` (must provide latitude, longitude, and radius)
 
+Use limit to specify the number of matched properties to return. Set auto_paginate to `True` to retrieve all results, this will override the limit.
+
 ```python
 results, filter_data = client.property_v2.search.retrieve(
     # parcl_ids=[5495449],
@@ -511,6 +513,8 @@ results, filter_data = client.property_v2.search.retrieve(
     max_record_added_date="2024-12-31",
     min_record_added_date="2024-12-13",
     property_types=["SINGLE_FAMILY", "CONDO", "TOWNHOUSE"],
+    limit=10,
+    # auto_paginate=True,
 )
 ```
 

parcllabs/__version__.py

@@ -1 +1 @@
-VERSION = "1.14.1"
+VERSION = "1.14.2"

parcllabs/enums.py

@@ -12,6 +12,7 @@ class RequestLimits(Enum):
     MAX_POST = 1000
     DEFAULT_SMALL = 1000
     DEFAULT_LARGE = 10000
+    PROPERTY_V2_MAX = 100000
 
 
 class ResponseCodes(Enum):

parcllabs/services/properties/property_v2.py

@@ -4,6 +4,7 @@
 
 import pandas as pd
 
+from parcllabs.enums import RequestLimits
 from parcllabs.services.parcllabs_service import ParclLabsService
 from parcllabs.services.validators import Validators
 
@@ -12,7 +13,9 @@ class PropertyV2Service(ParclLabsService):
     def __init__(self, *args: object, **kwargs: object) -> None:
         super().__init__(*args, **kwargs)
 
-    def _fetch_post(self, params: dict[str, Any], data: dict[str, Any]) -> list[dict]:
+    def _fetch_post(
+        self, params: dict[str, Any], data: dict[str, Any], auto_paginate: bool
+    ) -> list[dict]:
         """Fetch data using POST request with pagination support."""
         response = self._post(url=self.full_post_url, data=data, params=params)
         result = response.json()
@@ -22,7 +25,7 @@ def _fetch_post(self, params: dict[str, Any], data: dict[str, Any]) -> list[dict
         all_data = [result]
 
         # If we need to paginate, use concurrent requests
-        if pagination and pagination.get("has_more"):
+        if auto_paginate and pagination and pagination.get("has_more"):
             limit = pagination.get("limit")
             offset = pagination.get("offset")
             total_count = metadata.get("results", {}).get("total_available", 0)
@@ -261,6 +264,26 @@ def _build_owner_filters(self, **kwargs: dict) -> dict[str, Any]:
 
         return owner_filters
 
+    def _validate_limit(self, limit: int | None, auto_paginate: bool) -> int:
+        """Validate limit parameter."""
+        max_limit = RequestLimits.PROPERTY_V2_MAX.value
+
+        # If auto-paginate is enabled or no limit is provided, use maximum limit
+        if auto_paginate or limit is None:
+            if auto_paginate and limit is not None:
+                print(f"Auto-paginate is enabled. Setting limit to maximum value of {max_limit}.")
+            return max_limit
+
+        # If limit exceeds maximum, cap it
+        if limit > max_limit:
+            print(
+                f"Supplied limit value is too large for requested endpoint."
+                f"Setting limit to maximum value of {max_limit}."
+            )
+            return max_limit
+
+        return limit
+
     def retrieve(
         self,
         parcl_ids: list[int] | None = None,
@@ -290,7 +313,9 @@ def retrieve(
         owner_name: list[str] | None = None,
         is_investor_owned: bool | None = None,
         is_owner_occupied: bool | None = None,
-        params: Mapping[str, Any] | None = None,
+        limit: int | None = None,
+        params: Mapping[str, Any] | None = {},
+        auto_paginate: bool = False,
     ) -> tuple[pd.DataFrame, dict[str, Any]]:
         """
         Retrieve property data based on search criteria and filters.
@@ -353,8 +378,10 @@ def retrieve(
         data["event_filters"] = self._build_event_filters(**kwargs)
         data["owner_filters"] = self._build_owner_filters(**kwargs)
 
+        params["limit"] = self._validate_limit(limit, auto_paginate)
+
         # Make request with pagination
-        results = self._fetch_post(params=params or {}, data=data)
+        results = self._fetch_post(params=params, data=data, auto_paginate=auto_paginate)
 
         # Get metadata from results
         metadata = self._get_metadata(results)

tests/test_property_v2.py

@@ -2,6 +2,7 @@
 
 import pytest
 
+from parcllabs.enums import RequestLimits
 from parcllabs.services.properties.property_v2 import PropertyV2Service
 
 
@@ -139,12 +140,32 @@ def test_build_owner_filters(property_v2_service: PropertyV2Service) -> None:
     }
 
 
+def test_validate_limit(property_v2_service: PropertyV2Service) -> None:
+    assert (
+        property_v2_service._validate_limit(limit=None, auto_paginate=True)
+        == RequestLimits.PROPERTY_V2_MAX.value
+    )
+    assert (
+        property_v2_service._validate_limit(limit=None, auto_paginate=False)
+        == RequestLimits.PROPERTY_V2_MAX.value
+    )
+    assert (
+        property_v2_service._validate_limit(limit=100, auto_paginate=True)
+        == RequestLimits.PROPERTY_V2_MAX.value
+    )
+    assert property_v2_service._validate_limit(limit=100, auto_paginate=False) == 100
+    assert (
+        property_v2_service._validate_limit(limit=1000000000, auto_paginate=True)
+        == RequestLimits.PROPERTY_V2_MAX.value
+    )
+
+
 @patch.object(PropertyV2Service, "_post")
 def test_fetch_post_single_page(
     mock_post: Mock, property_v2_service: PropertyV2Service, mock_response: Mock
 ) -> None:
     mock_post.return_value = mock_response
-    result = property_v2_service._fetch_post(params={}, data={})
+    result = property_v2_service._fetch_post(params={}, data={}, auto_paginate=False)
 
     assert len(result) == 1
     assert result[0] == mock_response.json()
@@ -174,7 +195,7 @@ def test_fetch_post_pagination(mock_post: Mock, property_v2_service: PropertyV2S
     # Set up the mock to return different responses
     mock_post.side_effect = [first_response, second_response]
 
-    result = property_v2_service._fetch_post(params={}, data={})
+    result = property_v2_service._fetch_post(params={"limit": 1}, data={}, auto_paginate=True)
 
     assert len(result) == 2
     assert result[0]["data"][0]["parcl_id"] == 123
@@ -224,6 +245,7 @@ def test_retrieve(
         min_beds=2,
         max_beds=4,
         event_names=["LISTING"],
+        limit=10,
     )
     # check that the dataframe has the expected data
     assert len(df) == 1
@@ -234,7 +256,7 @@ def test_retrieve(
 
     # check that the correct data was passed to _fetch_post
     call_args = mock_fetch_post.call_args[1]
-    assert call_args["params"] == {}
+    assert call_args["params"] == {"limit": 10}
 
     data = call_args["data"]
     assert data["parcl_ids"] == [123]