From bb5a2600b67ddf9a1ab6c8748081a1ed3506c6e7 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 10 Sep 2025 11:38:49 +0200 Subject: [PATCH 1/8] Allow to define public api url different from api url --- src/apify_client/client.py | 21 +++++++++-- src/apify_client/clients/base/base_client.py | 12 ++++--- .../clients/resource_clients/dataset.py | 4 +-- .../resource_clients/key_value_store.py | 4 +-- tests/integration/conftest.py | 29 +++++++++------ tests/integration/test_dataset.py | 34 +++++++++++++++--- tests/integration/test_key_value_store.py | 36 ++++++++++++++++--- 7 files changed, 110 insertions(+), 30 deletions(-) diff --git a/src/apify_client/client.py b/src/apify_client/client.py index 648c65ff..c63d02a2 100644 --- a/src/apify_client/client.py +++ b/src/apify_client/client.py @@ -64,6 +64,7 @@ def __init__( token: str | None = None, *, api_url: str | None = None, + api_public_url: str | None = None, max_retries: int | None = 8, min_delay_between_retries_millis: int | None = 500, timeout_secs: int | None = DEFAULT_TIMEOUT, @@ -72,7 +73,10 @@ def __init__( Args: token: The Apify API token. - api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. + api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. It can + be internal url that is not globally accessible, in such case `api_public_url` should be set as well. + api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url` + is internal url that is not globally accessible. max_retries: How many times to retry a failed request at most. min_delay_between_retries_millis: How long will the client wait between retrying requests (increases exponentially from this value). @@ -81,6 +85,7 @@ def __init__( self.token = token api_url = (api_url or DEFAULT_API_URL).rstrip('/') self.base_url = f'{api_url}/{API_VERSION}' + self.public_base_url = (api_public_url or self.base_url).rstrip('/') self.max_retries = max_retries or 8 self.min_delay_between_retries_millis = min_delay_between_retries_millis or 500 self.timeout_secs = timeout_secs or DEFAULT_TIMEOUT @@ -103,6 +108,7 @@ def __init__( token: str | None = None, *, api_url: str | None = None, + api_public_url: str | None = None, max_retries: int | None = 8, min_delay_between_retries_millis: int | None = 500, timeout_secs: int | None = DEFAULT_TIMEOUT, @@ -111,7 +117,10 @@ def __init__( Args: token: The Apify API token. - api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. + api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. It can + be internal url that is not globally accessible, in such case `api_public_url` should be set as well. + api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url` + is internal url that is not globally accessible. max_retries: How many times to retry a failed request at most. min_delay_between_retries_millis: How long will the client wait between retrying requests (increases exponentially from this value). @@ -120,6 +129,7 @@ def __init__( super().__init__( token, api_url=api_url, + api_public_url=api_public_url, max_retries=max_retries, min_delay_between_retries_millis=min_delay_between_retries_millis, timeout_secs=timeout_secs, @@ -286,6 +296,7 @@ def __init__( token: str | None = None, *, api_url: str | None = None, + api_public_url: str | None = None, max_retries: int | None = 8, min_delay_between_retries_millis: int | None = 500, timeout_secs: int | None = DEFAULT_TIMEOUT, @@ -294,7 +305,10 @@ def __init__( Args: token: The Apify API token. - api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. + api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. It can + be internal url that is not globally accessible, in such case `api_public_url` should be set as well. + api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url` + is internal url that is not globally accessible. max_retries: How many times to retry a failed request at most. min_delay_between_retries_millis: How long will the client wait between retrying requests (increases exponentially from this value). @@ -303,6 +317,7 @@ def __init__( super().__init__( token, api_url=api_url, + api_public_url=api_public_url, max_retries=max_retries, min_delay_between_retries_millis=min_delay_between_retries_millis, timeout_secs=timeout_secs, diff --git a/src/apify_client/clients/base/base_client.py b/src/apify_client/clients/base/base_client.py index 2660c6cb..80ae7f1b 100644 --- a/src/apify_client/clients/base/base_client.py +++ b/src/apify_client/clients/base/base_client.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any +from urllib.parse import urlparse from apify_client._logging import WithLogDetailsClient from apify_client._utils import to_safe_id @@ -18,10 +19,13 @@ class _BaseBaseClient(metaclass=WithLogDetailsClient): http_client: HTTPClient | HTTPClientAsync root_client: ApifyClient | ApifyClientAsync - def _url(self, path: str | None = None) -> str: - if path is not None: - return f'{self.url}/{path}' - return self.url + def _url(self, path: str | None = None, *, public: bool = False) -> str: + url = f'{self.url}/{path}' if path is not None else self.url + + if public: + url_path = urlparse(url).path + return urlparse(self.root_client.public_base_url)._replace(path=url_path).geturl() + return url def _params(self, **kwargs: Any) -> dict: return { diff --git a/src/apify_client/clients/resource_clients/dataset.py b/src/apify_client/clients/resource_clients/dataset.py index 7d2babff..aadf592d 100644 --- a/src/apify_client/clients/resource_clients/dataset.py +++ b/src/apify_client/clients/resource_clients/dataset.py @@ -619,7 +619,7 @@ def create_items_public_url( ) request_params['signature'] = signature - items_public_url = urlparse(self._url('items')) + items_public_url = urlparse(self._url('items', public=True)) filtered_params = {k: v for k, v in request_params.items() if v is not None} if filtered_params: items_public_url = items_public_url._replace(query=urlencode(filtered_params)) @@ -1126,7 +1126,7 @@ async def create_items_public_url( ) request_params['signature'] = signature - items_public_url = urlparse(self._url('items')) + items_public_url = urlparse(self._url('items', public=True)) filtered_params = {k: v for k, v in request_params.items() if v is not None} if filtered_params: items_public_url = items_public_url._replace(query=urlencode(filtered_params)) diff --git a/src/apify_client/clients/resource_clients/key_value_store.py b/src/apify_client/clients/resource_clients/key_value_store.py index dcce7349..d124d985 100644 --- a/src/apify_client/clients/resource_clients/key_value_store.py +++ b/src/apify_client/clients/resource_clients/key_value_store.py @@ -307,7 +307,7 @@ def create_keys_public_url( ) request_params['signature'] = signature - keys_public_url = urlparse(self._url('keys')) + keys_public_url = urlparse(self._url('keys', public=True)) filtered_params = {k: v for k, v in request_params.items() if v is not None} if filtered_params: @@ -597,7 +597,7 @@ async def create_keys_public_url( ) request_params['signature'] = signature - keys_public_url = urlparse(self._url('keys')) + keys_public_url = urlparse(self._url('keys', public=True)) filtered_params = {k: v for k, v in request_params.items() if v is not None} if filtered_params: keys_public_url = keys_public_url._replace(query=urlencode(filtered_params)) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 5768e87f..d0c5bffb 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,15 +7,27 @@ TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' +parametrized_api_urls = pytest.mark.parametrize( + ('api_url', 'api_public_url'), + [ + ('https://api.apify.com', 'https://api.apify.com'), + ('https://api.apify.com', None), + ('https://api.apify.com', 'https://custom-public-url.com'), + ], +) -@pytest.fixture -def apify_client() -> ApifyClient: - api_token = os.getenv(TOKEN_ENV_VAR) - api_url = os.getenv(API_URL_ENV_VAR) - if not api_token: +@pytest.fixture +def api_token() -> str: + token = os.getenv(TOKEN_ENV_VAR) + if not token: raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!') + return token + +@pytest.fixture +def apify_client(api_token: str) -> ApifyClient: + api_url = os.getenv(API_URL_ENV_VAR) return ApifyClient(api_token, api_url=api_url) @@ -25,11 +37,6 @@ def apify_client() -> ApifyClient: # but `pytest-asyncio` closes the event loop after each test, # and uses a new one for the next test. @pytest.fixture -def apify_client_async() -> ApifyClientAsync: - api_token = os.getenv(TOKEN_ENV_VAR) +def apify_client_async(api_token: str) -> ApifyClientAsync: api_url = os.getenv(API_URL_ENV_VAR) - - if not api_token: - raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!') - return ApifyClientAsync(api_token, api_url=api_url) diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index 103f3d00..1446b6ad 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -1,13 +1,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING - import impit +from integration.conftest import parametrized_api_urls from integration.integration_test_utils import random_resource_name -if TYPE_CHECKING: - from apify_client import ApifyClient, ApifyClientAsync +from apify_client import ApifyClient, ApifyClientAsync class TestDatasetSync: @@ -47,6 +45,20 @@ def test_dataset_should_create_public_items_non_expiring_url(self, apify_client: dataset.delete() assert apify_client.dataset(created_dataset['id']).get() is None + @parametrized_api_urls + def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: + apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url) + created_store = apify_client.datasets().get_or_create(name=random_resource_name('key-value-store')) + dataset = apify_client.dataset(created_store['id']) + try: + public_url = dataset.create_items_public_url() + assert public_url == ( + f'{api_public_url or api_url}/v2/datasets/' + f'{created_store["id"]}/items?signature={public_url.split("signature=")[1]}' + ) + finally: + dataset.delete() + class TestDatasetAsync: async def test_dataset_should_create_public_items_expiring_url_with_params( @@ -88,3 +100,17 @@ async def test_dataset_should_create_public_items_non_expiring_url( await dataset.delete() assert await apify_client_async.dataset(created_dataset['id']).get() is None + + @parametrized_api_urls + async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: + apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url) + created_store = await apify_client.datasets().get_or_create(name=random_resource_name('key-value-store')) + dataset = apify_client.dataset(created_store['id']) + try: + public_url = await dataset.create_items_public_url() + assert public_url == ( + f'{api_public_url or api_url}/v2/datasets/' + f'{created_store["id"]}/items?signature={public_url.split("signature=")[1]}' + ) + finally: + await dataset.delete() diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index 38983835..83624b6a 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -1,13 +1,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING - import impit +from integration.conftest import parametrized_api_urls from integration.integration_test_utils import random_resource_name -if TYPE_CHECKING: - from apify_client import ApifyClient, ApifyClientAsync +from apify_client import ApifyClient, ApifyClientAsync class TestKeyValueStoreSync: @@ -47,6 +45,20 @@ def test_key_value_store_should_create_public_keys_non_expiring_url(self, apify_ store.delete() assert apify_client.key_value_store(created_store['id']).get() is None + @parametrized_api_urls + def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: + apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url) + created_store = apify_client.key_value_stores().get_or_create(name=random_resource_name('key-value-store')) + kvs = apify_client.key_value_store(created_store['id']) + try: + public_url = kvs.create_keys_public_url() + assert public_url == ( + f'{api_public_url or api_url}/v2/key-value-stores/' + f'{created_store["id"]}/keys?signature={public_url.split("signature=")[1]}' + ) + finally: + kvs.delete() + class TestKeyValueStoreAsync: async def test_key_value_store_should_create_expiring_keys_public_url_with_params( @@ -90,3 +102,19 @@ async def test_key_value_store_should_create_public_keys_non_expiring_url( await store.delete() assert await apify_client_async.key_value_store(created_store['id']).get() is None + + @parametrized_api_urls + async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: + apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url) + created_store = await apify_client.key_value_stores().get_or_create( + name=random_resource_name('key-value-store') + ) + kvs = apify_client.key_value_store(created_store['id']) + try: + public_url = await kvs.create_keys_public_url() + assert public_url == ( + f'{api_public_url or api_url}/v2/key-value-stores/' + f'{created_store["id"]}/keys?signature={public_url.split("signature=")[1]}' + ) + finally: + await kvs.delete() From 1db60088bed8a74674b6bda54b8be5939f6cddf2 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 10 Sep 2025 12:38:24 +0200 Subject: [PATCH 2/8] Add option to define custom public url --- src/apify_client/client.py | 2 +- src/apify_client/clients/base/base_client.py | 5 ++--- tests/integration/conftest.py | 2 ++ tests/integration/test_dataset.py | 5 +++-- tests/integration/test_key_value_store.py | 5 +++-- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/apify_client/client.py b/src/apify_client/client.py index c63d02a2..a91aa3a0 100644 --- a/src/apify_client/client.py +++ b/src/apify_client/client.py @@ -85,7 +85,7 @@ def __init__( self.token = token api_url = (api_url or DEFAULT_API_URL).rstrip('/') self.base_url = f'{api_url}/{API_VERSION}' - self.public_base_url = (api_public_url or self.base_url).rstrip('/') + self.public_base_url = (api_public_url or DEFAULT_API_URL).rstrip('/') self.max_retries = max_retries or 8 self.min_delay_between_retries_millis = min_delay_between_retries_millis or 500 self.timeout_secs = timeout_secs or DEFAULT_TIMEOUT diff --git a/src/apify_client/clients/base/base_client.py b/src/apify_client/clients/base/base_client.py index 80ae7f1b..00c8ff6f 100644 --- a/src/apify_client/clients/base/base_client.py +++ b/src/apify_client/clients/base/base_client.py @@ -1,7 +1,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any -from urllib.parse import urlparse +from urllib.parse import urljoin, urlparse from apify_client._logging import WithLogDetailsClient from apify_client._utils import to_safe_id @@ -23,8 +23,7 @@ def _url(self, path: str | None = None, *, public: bool = False) -> str: url = f'{self.url}/{path}' if path is not None else self.url if public: - url_path = urlparse(url).path - return urlparse(self.root_client.public_base_url)._replace(path=url_path).geturl() + return urljoin(self.root_client.public_base_url + '/', urlparse(url).path.strip('/')) return url def _params(self, **kwargs: Any) -> dict: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d0c5bffb..a5b61bba 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -13,6 +13,8 @@ ('https://api.apify.com', 'https://api.apify.com'), ('https://api.apify.com', None), ('https://api.apify.com', 'https://custom-public-url.com'), + ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path'), + ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path/'), ], ) diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index 1446b6ad..aaa08071 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -6,6 +6,7 @@ from integration.integration_test_utils import random_resource_name from apify_client import ApifyClient, ApifyClientAsync +from apify_client.client import DEFAULT_API_URL class TestDatasetSync: @@ -53,7 +54,7 @@ def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> try: public_url = dataset.create_items_public_url() assert public_url == ( - f'{api_public_url or api_url}/v2/datasets/' + f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/' f'{created_store["id"]}/items?signature={public_url.split("signature=")[1]}' ) finally: @@ -109,7 +110,7 @@ async def test_public_url(self, api_token: str, api_url: str, api_public_url: st try: public_url = await dataset.create_items_public_url() assert public_url == ( - f'{api_public_url or api_url}/v2/datasets/' + f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/' f'{created_store["id"]}/items?signature={public_url.split("signature=")[1]}' ) finally: diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index 83624b6a..688f2685 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -6,6 +6,7 @@ from integration.integration_test_utils import random_resource_name from apify_client import ApifyClient, ApifyClientAsync +from apify_client.client import DEFAULT_API_URL class TestKeyValueStoreSync: @@ -53,7 +54,7 @@ def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> try: public_url = kvs.create_keys_public_url() assert public_url == ( - f'{api_public_url or api_url}/v2/key-value-stores/' + f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/' f'{created_store["id"]}/keys?signature={public_url.split("signature=")[1]}' ) finally: @@ -113,7 +114,7 @@ async def test_public_url(self, api_token: str, api_url: str, api_public_url: st try: public_url = await kvs.create_keys_public_url() assert public_url == ( - f'{api_public_url or api_url}/v2/key-value-stores/' + f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/' f'{created_store["id"]}/keys?signature={public_url.split("signature=")[1]}' ) finally: From 6fb6f03e88ecdaee0f2ce5bc97627e8aeda50a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Proch=C3=A1zka?= Date: Wed, 10 Sep 2025 17:24:16 +0200 Subject: [PATCH 3/8] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jindřich Bär --- src/apify_client/client.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/apify_client/client.py b/src/apify_client/client.py index a91aa3a0..3cccc341 100644 --- a/src/apify_client/client.py +++ b/src/apify_client/client.py @@ -73,10 +73,10 @@ def __init__( Args: token: The Apify API token. - api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. It can - be internal url that is not globally accessible, in such case `api_public_url` should be set as well. + api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can + be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well. api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url` - is internal url that is not globally accessible. + is an internal URL that is not globally accessible. max_retries: How many times to retry a failed request at most. min_delay_between_retries_millis: How long will the client wait between retrying requests (increases exponentially from this value). @@ -117,10 +117,10 @@ def __init__( Args: token: The Apify API token. - api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. It can - be internal url that is not globally accessible, in such case `api_public_url` should be set as well. + api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can + be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well. api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url` - is internal url that is not globally accessible. + is an internal URL that is not globally accessible. max_retries: How many times to retry a failed request at most. min_delay_between_retries_millis: How long will the client wait between retrying requests (increases exponentially from this value). @@ -305,10 +305,10 @@ def __init__( Args: token: The Apify API token. - api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. It can - be internal url that is not globally accessible, in such case `api_public_url` should be set as well. + api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can + be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well. api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url` - is internal url that is not globally accessible. + is an internal URL that is not globally accessible. max_retries: How many times to retry a failed request at most. min_delay_between_retries_millis: How long will the client wait between retrying requests (increases exponentially from this value). From d7163b8f9d6c7f876290aeb00f765f3b623e46cd Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 10 Sep 2025 17:56:56 +0200 Subject: [PATCH 4/8] Add non existent url test --- tests/integration/test_dataset.py | 16 ++++++++++++++++ tests/integration/test_key_value_store.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index aaa08071..659624dd 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -60,6 +60,14 @@ def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> finally: dataset.delete() + def test_public_url_nonexistent_host(self, api_token: str) -> None: + dataset_name = 'whatever' + non_existent_url = 'http://10.0.88.214:8010' + apify_client = ApifyClient(token=api_token, api_url=non_existent_url) + kvs_client = apify_client.dataset(dataset_id=dataset_name) + assert kvs_client._url() == f'{non_existent_url}/v2/datasets/{dataset_name}' + assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/datasets/{dataset_name}' + class TestDatasetAsync: async def test_dataset_should_create_public_items_expiring_url_with_params( @@ -115,3 +123,11 @@ async def test_public_url(self, api_token: str, api_url: str, api_public_url: st ) finally: await dataset.delete() + + def test_public_url_nonexistent_host(self, api_token: str) -> None: + dataset_name = 'whatever' + non_existent_url = 'http://10.0.88.214:8010' + apify_client = ApifyClientAsync(token=api_token, api_url=non_existent_url) + kvs_client = apify_client.dataset(dataset_id=dataset_name) + assert kvs_client._url() == f'{non_existent_url}/v2/datasets/{dataset_name}' + assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/datasets/{dataset_name}' diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index 688f2685..e5c8d4ef 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -60,6 +60,14 @@ def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> finally: kvs.delete() + def test_public_url_nonexistent_host(self, api_token: str) -> None: + kvs_name = 'whatever' + non_existent_url = 'http://10.0.88.214:8010' + apify_client = ApifyClient(token=api_token, api_url=non_existent_url) + kvs_client = apify_client.key_value_store(key_value_store_id=kvs_name) + assert kvs_client._url() == f'{non_existent_url}/v2/key-value-stores/{kvs_name}' + assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/key-value-stores/{kvs_name}' + class TestKeyValueStoreAsync: async def test_key_value_store_should_create_expiring_keys_public_url_with_params( @@ -119,3 +127,11 @@ async def test_public_url(self, api_token: str, api_url: str, api_public_url: st ) finally: await kvs.delete() + + async def test_public_url_nonexistent_host(self, api_token: str) -> None: + kvs_name = 'whatever' + non_existent_url = 'http://10.0.88.214:8010' + apify_client = ApifyClientAsync(token=api_token, api_url=non_existent_url) + kvs_client = apify_client.key_value_store(key_value_store_id=kvs_name) + assert kvs_client._url() == f'{non_existent_url}/v2/key-value-stores/{kvs_name}' + assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/key-value-stores/{kvs_name}' From 77c5f200c184f3a6b6d06fccdd2f2e34b8a50dc4 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 10 Sep 2025 18:05:13 +0200 Subject: [PATCH 5/8] Treat url and public url the same way with respect to the v2 part of the path --- src/apify_client/client.py | 3 ++- src/apify_client/clients/base/base_client.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/apify_client/client.py b/src/apify_client/client.py index 3cccc341..b6ed7abf 100644 --- a/src/apify_client/client.py +++ b/src/apify_client/client.py @@ -85,7 +85,8 @@ def __init__( self.token = token api_url = (api_url or DEFAULT_API_URL).rstrip('/') self.base_url = f'{api_url}/{API_VERSION}' - self.public_base_url = (api_public_url or DEFAULT_API_URL).rstrip('/') + api_public_url = (api_public_url or DEFAULT_API_URL).rstrip('/') + self.public_base_url = f'{api_public_url}/{API_VERSION}' self.max_retries = max_retries or 8 self.min_delay_between_retries_millis = min_delay_between_retries_millis or 500 self.timeout_secs = timeout_secs or DEFAULT_TIMEOUT diff --git a/src/apify_client/clients/base/base_client.py b/src/apify_client/clients/base/base_client.py index 00c8ff6f..370735a3 100644 --- a/src/apify_client/clients/base/base_client.py +++ b/src/apify_client/clients/base/base_client.py @@ -1,7 +1,6 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any -from urllib.parse import urljoin, urlparse from apify_client._logging import WithLogDetailsClient from apify_client._utils import to_safe_id @@ -23,7 +22,7 @@ def _url(self, path: str | None = None, *, public: bool = False) -> str: url = f'{self.url}/{path}' if path is not None else self.url if public: - return urljoin(self.root_client.public_base_url + '/', urlparse(url).path.strip('/')) + return url.replace(self.root_client.base_url, self.root_client.public_base_url) return url def _params(self, **kwargs: Any) -> dict: From a28026c3706764bd80b00ec2d943231d758a2160 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 11 Sep 2025 11:05:32 +0200 Subject: [PATCH 6/8] Refactor the tests to avoid real API calls --- tests/integration/conftest.py | 1 + tests/integration/test_dataset.py | 67 ++++++++++++---------- tests/integration/test_key_value_store.py | 68 +++++++++++++---------- 3 files changed, 78 insertions(+), 58 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a5b61bba..e46a25de 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -15,6 +15,7 @@ ('https://api.apify.com', 'https://custom-public-url.com'), ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path'), ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path/'), + ('http://10.0.88.214:8010', 'https://api.apify.com'), ], ) diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index 659624dd..40cfa41f 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -1,5 +1,8 @@ from __future__ import annotations +from unittest import mock +from unittest.mock import Mock + import impit from integration.conftest import parametrized_api_urls @@ -8,6 +11,32 @@ from apify_client import ApifyClient, ApifyClientAsync from apify_client.client import DEFAULT_API_URL +MOCKED_API_DATASET_RESPONSE = """{ + "data": { + "id": "someID", + "name": "name", + "userId": "userId", + "createdAt": "2025-09-11T08:48:51.806Z", + "modifiedAt": "2025-09-11T08:48:51.806Z", + "accessedAt": "2025-09-11T08:48:51.806Z", + "actId": null, + "actRunId": null, + "schema": null, + "stats": { + "readCount": 0, + "writeCount": 0, + "deleteCount": 0, + "listCount": 0, + "storageBytes": 0 + }, + "fields": [], + "consoleUrl": "https://console.apify.com/storage/datasets/someID", + "itemsPublicUrl": "https://api.apify.com/v2/datasets/someID/items", + "generalAccess": "FOLLOW_USER_SETTING", + "urlSigningSecretKey": "urlSigningSecretKey" + } +}""" + class TestDatasetSync: def test_dataset_should_create_public_items_expiring_url_with_params(self, apify_client: ApifyClient) -> None: @@ -49,24 +78,15 @@ def test_dataset_should_create_public_items_non_expiring_url(self, apify_client: @parametrized_api_urls def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url) - created_store = apify_client.datasets().get_or_create(name=random_resource_name('key-value-store')) - dataset = apify_client.dataset(created_store['id']) - try: + dataset = apify_client.dataset('someID') + + # Mock the API call to return predefined response + with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_DATASET_RESPONSE)): public_url = dataset.create_items_public_url() assert public_url == ( f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/' - f'{created_store["id"]}/items?signature={public_url.split("signature=")[1]}' + f'someID/items?signature={public_url.split("signature=")[1]}' ) - finally: - dataset.delete() - - def test_public_url_nonexistent_host(self, api_token: str) -> None: - dataset_name = 'whatever' - non_existent_url = 'http://10.0.88.214:8010' - apify_client = ApifyClient(token=api_token, api_url=non_existent_url) - kvs_client = apify_client.dataset(dataset_id=dataset_name) - assert kvs_client._url() == f'{non_existent_url}/v2/datasets/{dataset_name}' - assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/datasets/{dataset_name}' class TestDatasetAsync: @@ -113,21 +133,12 @@ async def test_dataset_should_create_public_items_non_expiring_url( @parametrized_api_urls async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url) - created_store = await apify_client.datasets().get_or_create(name=random_resource_name('key-value-store')) - dataset = apify_client.dataset(created_store['id']) - try: + dataset = apify_client.dataset('someID') + + # Mock the API call to return predefined response + with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_DATASET_RESPONSE)): public_url = await dataset.create_items_public_url() assert public_url == ( f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/' - f'{created_store["id"]}/items?signature={public_url.split("signature=")[1]}' + f'someID/items?signature={public_url.split("signature=")[1]}' ) - finally: - await dataset.delete() - - def test_public_url_nonexistent_host(self, api_token: str) -> None: - dataset_name = 'whatever' - non_existent_url = 'http://10.0.88.214:8010' - apify_client = ApifyClientAsync(token=api_token, api_url=non_existent_url) - kvs_client = apify_client.dataset(dataset_id=dataset_name) - assert kvs_client._url() == f'{non_existent_url}/v2/datasets/{dataset_name}' - assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/datasets/{dataset_name}' diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index e5c8d4ef..dc2c07fe 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -1,5 +1,8 @@ from __future__ import annotations +from unittest import mock +from unittest.mock import Mock + import impit from integration.conftest import parametrized_api_urls @@ -8,6 +11,31 @@ from apify_client import ApifyClient, ApifyClientAsync from apify_client.client import DEFAULT_API_URL +MOCKED_API_KVS_RESPONSE = """{ + "data": { + "id": "someID", + "name": "name", + "userId": "userId", + "createdAt": "2025-09-11T08:48:51.806Z", + "modifiedAt": "2025-09-11T08:48:51.806Z", + "accessedAt": "2025-09-11T08:48:51.806Z", + "actId": null, + "actRunId": null, + "schema": null, + "stats": { + "readCount": 0, + "writeCount": 0, + "deleteCount": 0, + "listCount": 0, + "storageBytes": 0 + }, + "consoleUrl": "https://console.apify.com/storage/key-value-stores/someID", + "keysPublicUrl": "https://api.apify.com/v2/key-value-stores/someID/keys", + "generalAccess": "FOLLOW_USER_SETTING", + "urlSigningSecretKey": "urlSigningSecretKey" + } +}""" + class TestKeyValueStoreSync: def test_key_value_store_should_create_expiring_keys_public_url_with_params( @@ -49,24 +77,15 @@ def test_key_value_store_should_create_public_keys_non_expiring_url(self, apify_ @parametrized_api_urls def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url) - created_store = apify_client.key_value_stores().get_or_create(name=random_resource_name('key-value-store')) - kvs = apify_client.key_value_store(created_store['id']) - try: + kvs = apify_client.key_value_store('someID') + + # Mock the API call to return predefined response + with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_KVS_RESPONSE)): public_url = kvs.create_keys_public_url() assert public_url == ( f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/' - f'{created_store["id"]}/keys?signature={public_url.split("signature=")[1]}' + f'someID/keys?signature={public_url.split("signature=")[1]}' ) - finally: - kvs.delete() - - def test_public_url_nonexistent_host(self, api_token: str) -> None: - kvs_name = 'whatever' - non_existent_url = 'http://10.0.88.214:8010' - apify_client = ApifyClient(token=api_token, api_url=non_existent_url) - kvs_client = apify_client.key_value_store(key_value_store_id=kvs_name) - assert kvs_client._url() == f'{non_existent_url}/v2/key-value-stores/{kvs_name}' - assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/key-value-stores/{kvs_name}' class TestKeyValueStoreAsync: @@ -115,23 +134,12 @@ async def test_key_value_store_should_create_public_keys_non_expiring_url( @parametrized_api_urls async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url) - created_store = await apify_client.key_value_stores().get_or_create( - name=random_resource_name('key-value-store') - ) - kvs = apify_client.key_value_store(created_store['id']) - try: + kvs = apify_client.key_value_store('someID') + + # Mock the API call to return predefined response + with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_KVS_RESPONSE)): public_url = await kvs.create_keys_public_url() assert public_url == ( f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/' - f'{created_store["id"]}/keys?signature={public_url.split("signature=")[1]}' + f'someID/keys?signature={public_url.split("signature=")[1]}' ) - finally: - await kvs.delete() - - async def test_public_url_nonexistent_host(self, api_token: str) -> None: - kvs_name = 'whatever' - non_existent_url = 'http://10.0.88.214:8010' - apify_client = ApifyClientAsync(token=api_token, api_url=non_existent_url) - kvs_client = apify_client.key_value_store(key_value_store_id=kvs_name) - assert kvs_client._url() == f'{non_existent_url}/v2/key-value-stores/{kvs_name}' - assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/key-value-stores/{kvs_name}' From 116621b5d1a5ff62316133541143f9ab63a89059 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Fri, 12 Sep 2025 09:31:25 +0200 Subject: [PATCH 7/8] Update tests based on review --- src/apify_client/clients/base/base_client.py | 4 +++- tests/integration/conftest.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/apify_client/clients/base/base_client.py b/src/apify_client/clients/base/base_client.py index 370735a3..c5aa744c 100644 --- a/src/apify_client/clients/base/base_client.py +++ b/src/apify_client/clients/base/base_client.py @@ -22,7 +22,9 @@ def _url(self, path: str | None = None, *, public: bool = False) -> str: url = f'{self.url}/{path}' if path is not None else self.url if public: - return url.replace(self.root_client.base_url, self.root_client.public_base_url) + if not url.startswith(self.root_client.base_url): + raise ValueError('API based URL has to start with `self.root_client.base_url`') + return url.replace(self.root_client.base_url, self.root_client.public_base_url, 1) return url def _params(self, **kwargs: Any) -> dict: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e46a25de..e31456ea 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -16,6 +16,7 @@ ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path'), ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path/'), ('http://10.0.88.214:8010', 'https://api.apify.com'), + ('http://10.0.88.214:8010', None), ], ) From 2d31a37ace0e5dacf23e997cdf2ff5306db339b0 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Fri, 12 Sep 2025 13:10:44 +0200 Subject: [PATCH 8/8] Move shared parametrized_api_urls to util file --- tests/integration/conftest.py | 13 ------------- tests/integration/integration_test_utils.py | 16 ++++++++++++++++ tests/integration/test_dataset.py | 3 +-- tests/integration/test_key_value_store.py | 3 +-- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e31456ea..79e9e397 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -7,19 +7,6 @@ TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' -parametrized_api_urls = pytest.mark.parametrize( - ('api_url', 'api_public_url'), - [ - ('https://api.apify.com', 'https://api.apify.com'), - ('https://api.apify.com', None), - ('https://api.apify.com', 'https://custom-public-url.com'), - ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path'), - ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path/'), - ('http://10.0.88.214:8010', 'https://api.apify.com'), - ('http://10.0.88.214:8010', None), - ], -) - @pytest.fixture def api_token() -> str: diff --git a/tests/integration/integration_test_utils.py b/tests/integration/integration_test_utils.py index 8e8494e8..28705761 100644 --- a/tests/integration/integration_test_utils.py +++ b/tests/integration/integration_test_utils.py @@ -1,6 +1,8 @@ import secrets import string +import pytest + def random_string(length: int = 10) -> str: return ''.join(secrets.choice(string.ascii_letters) for _ in range(length)) @@ -8,3 +10,17 @@ def random_string(length: int = 10) -> str: def random_resource_name(resource: str) -> str: return f'python-client-test-{resource}-{random_string(5)}' + + +parametrized_api_urls = pytest.mark.parametrize( + ('api_url', 'api_public_url'), + [ + ('https://api.apify.com', 'https://api.apify.com'), + ('https://api.apify.com', None), + ('https://api.apify.com', 'https://custom-public-url.com'), + ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path'), + ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path/'), + ('http://10.0.88.214:8010', 'https://api.apify.com'), + ('http://10.0.88.214:8010', None), + ], +) diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index 40cfa41f..03a6e56f 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -5,8 +5,7 @@ import impit -from integration.conftest import parametrized_api_urls -from integration.integration_test_utils import random_resource_name +from integration.integration_test_utils import parametrized_api_urls, random_resource_name from apify_client import ApifyClient, ApifyClientAsync from apify_client.client import DEFAULT_API_URL diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index dc2c07fe..3f33013d 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -5,8 +5,7 @@ import impit -from integration.conftest import parametrized_api_urls -from integration.integration_test_utils import random_resource_name +from integration.integration_test_utils import parametrized_api_urls, random_resource_name from apify_client import ApifyClient, ApifyClientAsync from apify_client.client import DEFAULT_API_URL