diff --git a/CHANGELOG.md b/CHANGELOG.md index 44ea91e..37a0072 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ +### v1.14.2 +- Add autopaginate support for `property_v2.search` endpoint, increase default limit to 100,000. + ### v1.14.1 - Update `property_v2.search` endpoint params. +### v1.14.0 +- Simplification of parameter validation, reduce complexity of code. + ### v1.13.0 - Add support for `property_v2.search` endpoint. diff --git a/README.md b/README.md index 6188869..cafb070 100644 --- a/README.md +++ b/README.md @@ -482,11 +482,13 @@ addresses = client.property_address.search.retrieve( ### Property Search V2 -Gets a list of unique properties and their associated metadata and events based on a set of filters. Use one of three search methods: +Gets a list of unique properties and their associated metadata and events based on a set of property, event, and owner filters. Use one of three search methods: 1. `parcl_ids` 2. `parcl_property_ids` 3. `geo_coordinates` (must provide latitude, longitude, and radius) +Use limit to specify the number of matched properties to return. Set auto_paginate to `True` to retrieve all results, this will override the limit. + ```python results, filter_data = client.property_v2.search.retrieve( # parcl_ids=[5495449], @@ -511,6 +513,8 @@ results, filter_data = client.property_v2.search.retrieve( max_record_added_date="2024-12-31", min_record_added_date="2024-12-13", property_types=["SINGLE_FAMILY", "CONDO", "TOWNHOUSE"], + limit=10, + # auto_paginate=True, ) ``` diff --git a/parcllabs/__version__.py b/parcllabs/__version__.py index ffeef94..ed0a470 100644 --- a/parcllabs/__version__.py +++ b/parcllabs/__version__.py @@ -1 +1 @@ -VERSION = "1.14.1" +VERSION = "1.14.2" diff --git a/parcllabs/enums.py b/parcllabs/enums.py index 9041abb..a54879d 100644 --- a/parcllabs/enums.py +++ b/parcllabs/enums.py @@ -12,6 +12,7 @@ class RequestLimits(Enum): MAX_POST = 1000 DEFAULT_SMALL = 1000 DEFAULT_LARGE = 10000 + PROPERTY_V2_MAX = 100000 class ResponseCodes(Enum): diff --git a/parcllabs/services/properties/property_v2.py b/parcllabs/services/properties/property_v2.py index 558b255..43a1237 100644 --- a/parcllabs/services/properties/property_v2.py +++ b/parcllabs/services/properties/property_v2.py @@ -4,6 +4,7 @@ import pandas as pd +from parcllabs.enums import RequestLimits from parcllabs.services.parcllabs_service import ParclLabsService from parcllabs.services.validators import Validators @@ -12,7 +13,9 @@ class PropertyV2Service(ParclLabsService): def __init__(self, *args: object, **kwargs: object) -> None: super().__init__(*args, **kwargs) - def _fetch_post(self, params: dict[str, Any], data: dict[str, Any]) -> list[dict]: + def _fetch_post( + self, params: dict[str, Any], data: dict[str, Any], auto_paginate: bool + ) -> list[dict]: """Fetch data using POST request with pagination support.""" response = self._post(url=self.full_post_url, data=data, params=params) result = response.json() @@ -22,7 +25,7 @@ def _fetch_post(self, params: dict[str, Any], data: dict[str, Any]) -> list[dict all_data = [result] # If we need to paginate, use concurrent requests - if pagination and pagination.get("has_more"): + if auto_paginate and pagination and pagination.get("has_more"): limit = pagination.get("limit") offset = pagination.get("offset") total_count = metadata.get("results", {}).get("total_available", 0) @@ -261,6 +264,26 @@ def _build_owner_filters(self, **kwargs: dict) -> dict[str, Any]: return owner_filters + def _validate_limit(self, limit: int | None, auto_paginate: bool) -> int: + """Validate limit parameter.""" + max_limit = RequestLimits.PROPERTY_V2_MAX.value + + # If auto-paginate is enabled or no limit is provided, use maximum limit + if auto_paginate or limit is None: + if auto_paginate and limit is not None: + print(f"Auto-paginate is enabled. Setting limit to maximum value of {max_limit}.") + return max_limit + + # If limit exceeds maximum, cap it + if limit > max_limit: + print( + f"Supplied limit value is too large for requested endpoint." + f"Setting limit to maximum value of {max_limit}." + ) + return max_limit + + return limit + def retrieve( self, parcl_ids: list[int] | None = None, @@ -290,7 +313,9 @@ def retrieve( owner_name: list[str] | None = None, is_investor_owned: bool | None = None, is_owner_occupied: bool | None = None, - params: Mapping[str, Any] | None = None, + limit: int | None = None, + params: Mapping[str, Any] | None = {}, + auto_paginate: bool = False, ) -> tuple[pd.DataFrame, dict[str, Any]]: """ Retrieve property data based on search criteria and filters. @@ -353,8 +378,10 @@ def retrieve( data["event_filters"] = self._build_event_filters(**kwargs) data["owner_filters"] = self._build_owner_filters(**kwargs) + params["limit"] = self._validate_limit(limit, auto_paginate) + # Make request with pagination - results = self._fetch_post(params=params or {}, data=data) + results = self._fetch_post(params=params, data=data, auto_paginate=auto_paginate) # Get metadata from results metadata = self._get_metadata(results) diff --git a/tests/test_property_v2.py b/tests/test_property_v2.py index b242e2f..66a9a2a 100644 --- a/tests/test_property_v2.py +++ b/tests/test_property_v2.py @@ -2,6 +2,7 @@ import pytest +from parcllabs.enums import RequestLimits from parcllabs.services.properties.property_v2 import PropertyV2Service @@ -139,12 +140,32 @@ def test_build_owner_filters(property_v2_service: PropertyV2Service) -> None: } +def test_validate_limit(property_v2_service: PropertyV2Service) -> None: + assert ( + property_v2_service._validate_limit(limit=None, auto_paginate=True) + == RequestLimits.PROPERTY_V2_MAX.value + ) + assert ( + property_v2_service._validate_limit(limit=None, auto_paginate=False) + == RequestLimits.PROPERTY_V2_MAX.value + ) + assert ( + property_v2_service._validate_limit(limit=100, auto_paginate=True) + == RequestLimits.PROPERTY_V2_MAX.value + ) + assert property_v2_service._validate_limit(limit=100, auto_paginate=False) == 100 + assert ( + property_v2_service._validate_limit(limit=1000000000, auto_paginate=True) + == RequestLimits.PROPERTY_V2_MAX.value + ) + + @patch.object(PropertyV2Service, "_post") def test_fetch_post_single_page( mock_post: Mock, property_v2_service: PropertyV2Service, mock_response: Mock ) -> None: mock_post.return_value = mock_response - result = property_v2_service._fetch_post(params={}, data={}) + result = property_v2_service._fetch_post(params={}, data={}, auto_paginate=False) assert len(result) == 1 assert result[0] == mock_response.json() @@ -174,7 +195,7 @@ def test_fetch_post_pagination(mock_post: Mock, property_v2_service: PropertyV2S # Set up the mock to return different responses mock_post.side_effect = [first_response, second_response] - result = property_v2_service._fetch_post(params={}, data={}) + result = property_v2_service._fetch_post(params={"limit": 1}, data={}, auto_paginate=True) assert len(result) == 2 assert result[0]["data"][0]["parcl_id"] == 123 @@ -224,6 +245,7 @@ def test_retrieve( min_beds=2, max_beds=4, event_names=["LISTING"], + limit=10, ) # check that the dataframe has the expected data assert len(df) == 1 @@ -234,7 +256,7 @@ def test_retrieve( # check that the correct data was passed to _fetch_post call_args = mock_fetch_post.call_args[1] - assert call_args["params"] == {} + assert call_args["params"] == {"limit": 10} data = call_args["data"] assert data["parcl_ids"] == [123]