diff --git a/src/apify_client/client.py b/src/apify_client/client.py index 648c65ff..b6ed7abf 100644 --- a/src/apify_client/client.py +++ b/src/apify_client/client.py @@ -64,6 +64,7 @@ def __init__( token: str | None = None, *, api_url: str | None = None, + api_public_url: str | None = None, max_retries: int | None = 8, min_delay_between_retries_millis: int | None = 500, timeout_secs: int | None = DEFAULT_TIMEOUT, @@ -72,7 +73,10 @@ def __init__( Args: token: The Apify API token. - api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. + api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can + be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well. + api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url` + is an internal URL that is not globally accessible. max_retries: How many times to retry a failed request at most. min_delay_between_retries_millis: How long will the client wait between retrying requests (increases exponentially from this value). @@ -81,6 +85,8 @@ def __init__( self.token = token api_url = (api_url or DEFAULT_API_URL).rstrip('/') self.base_url = f'{api_url}/{API_VERSION}' + api_public_url = (api_public_url or DEFAULT_API_URL).rstrip('/') + self.public_base_url = f'{api_public_url}/{API_VERSION}' self.max_retries = max_retries or 8 self.min_delay_between_retries_millis = min_delay_between_retries_millis or 500 self.timeout_secs = timeout_secs or DEFAULT_TIMEOUT @@ -103,6 +109,7 @@ def __init__( token: str | None = None, *, api_url: str | None = None, + api_public_url: str | None = None, max_retries: int | None = 8, min_delay_between_retries_millis: int | None = 500, timeout_secs: int | None = DEFAULT_TIMEOUT, @@ -111,7 +118,10 @@ def __init__( Args: token: The Apify API token. - api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. + api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can + be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well. + api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url` + is an internal URL that is not globally accessible. max_retries: How many times to retry a failed request at most. min_delay_between_retries_millis: How long will the client wait between retrying requests (increases exponentially from this value). @@ -120,6 +130,7 @@ def __init__( super().__init__( token, api_url=api_url, + api_public_url=api_public_url, max_retries=max_retries, min_delay_between_retries_millis=min_delay_between_retries_millis, timeout_secs=timeout_secs, @@ -286,6 +297,7 @@ def __init__( token: str | None = None, *, api_url: str | None = None, + api_public_url: str | None = None, max_retries: int | None = 8, min_delay_between_retries_millis: int | None = 500, timeout_secs: int | None = DEFAULT_TIMEOUT, @@ -294,7 +306,10 @@ def __init__( Args: token: The Apify API token. - api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com. + api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can + be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well. + api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url` + is an internal URL that is not globally accessible. max_retries: How many times to retry a failed request at most. min_delay_between_retries_millis: How long will the client wait between retrying requests (increases exponentially from this value). @@ -303,6 +318,7 @@ def __init__( super().__init__( token, api_url=api_url, + api_public_url=api_public_url, max_retries=max_retries, min_delay_between_retries_millis=min_delay_between_retries_millis, timeout_secs=timeout_secs, diff --git a/src/apify_client/clients/base/base_client.py b/src/apify_client/clients/base/base_client.py index 2660c6cb..c5aa744c 100644 --- a/src/apify_client/clients/base/base_client.py +++ b/src/apify_client/clients/base/base_client.py @@ -18,10 +18,14 @@ class _BaseBaseClient(metaclass=WithLogDetailsClient): http_client: HTTPClient | HTTPClientAsync root_client: ApifyClient | ApifyClientAsync - def _url(self, path: str | None = None) -> str: - if path is not None: - return f'{self.url}/{path}' - return self.url + def _url(self, path: str | None = None, *, public: bool = False) -> str: + url = f'{self.url}/{path}' if path is not None else self.url + + if public: + if not url.startswith(self.root_client.base_url): + raise ValueError('API based URL has to start with `self.root_client.base_url`') + return url.replace(self.root_client.base_url, self.root_client.public_base_url, 1) + return url def _params(self, **kwargs: Any) -> dict: return { diff --git a/src/apify_client/clients/resource_clients/dataset.py b/src/apify_client/clients/resource_clients/dataset.py index 7d2babff..aadf592d 100644 --- a/src/apify_client/clients/resource_clients/dataset.py +++ b/src/apify_client/clients/resource_clients/dataset.py @@ -619,7 +619,7 @@ def create_items_public_url( ) request_params['signature'] = signature - items_public_url = urlparse(self._url('items')) + items_public_url = urlparse(self._url('items', public=True)) filtered_params = {k: v for k, v in request_params.items() if v is not None} if filtered_params: items_public_url = items_public_url._replace(query=urlencode(filtered_params)) @@ -1126,7 +1126,7 @@ async def create_items_public_url( ) request_params['signature'] = signature - items_public_url = urlparse(self._url('items')) + items_public_url = urlparse(self._url('items', public=True)) filtered_params = {k: v for k, v in request_params.items() if v is not None} if filtered_params: items_public_url = items_public_url._replace(query=urlencode(filtered_params)) diff --git a/src/apify_client/clients/resource_clients/key_value_store.py b/src/apify_client/clients/resource_clients/key_value_store.py index dcce7349..d124d985 100644 --- a/src/apify_client/clients/resource_clients/key_value_store.py +++ b/src/apify_client/clients/resource_clients/key_value_store.py @@ -307,7 +307,7 @@ def create_keys_public_url( ) request_params['signature'] = signature - keys_public_url = urlparse(self._url('keys')) + keys_public_url = urlparse(self._url('keys', public=True)) filtered_params = {k: v for k, v in request_params.items() if v is not None} if filtered_params: @@ -597,7 +597,7 @@ async def create_keys_public_url( ) request_params['signature'] = signature - keys_public_url = urlparse(self._url('keys')) + keys_public_url = urlparse(self._url('keys', public=True)) filtered_params = {k: v for k, v in request_params.items() if v is not None} if filtered_params: keys_public_url = keys_public_url._replace(query=urlencode(filtered_params)) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 5768e87f..79e9e397 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -9,13 +9,16 @@ @pytest.fixture -def apify_client() -> ApifyClient: - api_token = os.getenv(TOKEN_ENV_VAR) - api_url = os.getenv(API_URL_ENV_VAR) - - if not api_token: +def api_token() -> str: + token = os.getenv(TOKEN_ENV_VAR) + if not token: raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!') + return token + +@pytest.fixture +def apify_client(api_token: str) -> ApifyClient: + api_url = os.getenv(API_URL_ENV_VAR) return ApifyClient(api_token, api_url=api_url) @@ -25,11 +28,6 @@ def apify_client() -> ApifyClient: # but `pytest-asyncio` closes the event loop after each test, # and uses a new one for the next test. @pytest.fixture -def apify_client_async() -> ApifyClientAsync: - api_token = os.getenv(TOKEN_ENV_VAR) +def apify_client_async(api_token: str) -> ApifyClientAsync: api_url = os.getenv(API_URL_ENV_VAR) - - if not api_token: - raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!') - return ApifyClientAsync(api_token, api_url=api_url) diff --git a/tests/integration/integration_test_utils.py b/tests/integration/integration_test_utils.py index 8e8494e8..28705761 100644 --- a/tests/integration/integration_test_utils.py +++ b/tests/integration/integration_test_utils.py @@ -1,6 +1,8 @@ import secrets import string +import pytest + def random_string(length: int = 10) -> str: return ''.join(secrets.choice(string.ascii_letters) for _ in range(length)) @@ -8,3 +10,17 @@ def random_string(length: int = 10) -> str: def random_resource_name(resource: str) -> str: return f'python-client-test-{resource}-{random_string(5)}' + + +parametrized_api_urls = pytest.mark.parametrize( + ('api_url', 'api_public_url'), + [ + ('https://api.apify.com', 'https://api.apify.com'), + ('https://api.apify.com', None), + ('https://api.apify.com', 'https://custom-public-url.com'), + ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path'), + ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path/'), + ('http://10.0.88.214:8010', 'https://api.apify.com'), + ('http://10.0.88.214:8010', None), + ], +) diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index 103f3d00..03a6e56f 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -1,13 +1,40 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from unittest import mock +from unittest.mock import Mock import impit -from integration.integration_test_utils import random_resource_name - -if TYPE_CHECKING: - from apify_client import ApifyClient, ApifyClientAsync +from integration.integration_test_utils import parametrized_api_urls, random_resource_name + +from apify_client import ApifyClient, ApifyClientAsync +from apify_client.client import DEFAULT_API_URL + +MOCKED_API_DATASET_RESPONSE = """{ + "data": { + "id": "someID", + "name": "name", + "userId": "userId", + "createdAt": "2025-09-11T08:48:51.806Z", + "modifiedAt": "2025-09-11T08:48:51.806Z", + "accessedAt": "2025-09-11T08:48:51.806Z", + "actId": null, + "actRunId": null, + "schema": null, + "stats": { + "readCount": 0, + "writeCount": 0, + "deleteCount": 0, + "listCount": 0, + "storageBytes": 0 + }, + "fields": [], + "consoleUrl": "https://console.apify.com/storage/datasets/someID", + "itemsPublicUrl": "https://api.apify.com/v2/datasets/someID/items", + "generalAccess": "FOLLOW_USER_SETTING", + "urlSigningSecretKey": "urlSigningSecretKey" + } +}""" class TestDatasetSync: @@ -47,6 +74,19 @@ def test_dataset_should_create_public_items_non_expiring_url(self, apify_client: dataset.delete() assert apify_client.dataset(created_dataset['id']).get() is None + @parametrized_api_urls + def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: + apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url) + dataset = apify_client.dataset('someID') + + # Mock the API call to return predefined response + with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_DATASET_RESPONSE)): + public_url = dataset.create_items_public_url() + assert public_url == ( + f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/' + f'someID/items?signature={public_url.split("signature=")[1]}' + ) + class TestDatasetAsync: async def test_dataset_should_create_public_items_expiring_url_with_params( @@ -88,3 +128,16 @@ async def test_dataset_should_create_public_items_non_expiring_url( await dataset.delete() assert await apify_client_async.dataset(created_dataset['id']).get() is None + + @parametrized_api_urls + async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: + apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url) + dataset = apify_client.dataset('someID') + + # Mock the API call to return predefined response + with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_DATASET_RESPONSE)): + public_url = await dataset.create_items_public_url() + assert public_url == ( + f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/' + f'someID/items?signature={public_url.split("signature=")[1]}' + ) diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index 38983835..3f33013d 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -1,13 +1,39 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from unittest import mock +from unittest.mock import Mock import impit -from integration.integration_test_utils import random_resource_name - -if TYPE_CHECKING: - from apify_client import ApifyClient, ApifyClientAsync +from integration.integration_test_utils import parametrized_api_urls, random_resource_name + +from apify_client import ApifyClient, ApifyClientAsync +from apify_client.client import DEFAULT_API_URL + +MOCKED_API_KVS_RESPONSE = """{ + "data": { + "id": "someID", + "name": "name", + "userId": "userId", + "createdAt": "2025-09-11T08:48:51.806Z", + "modifiedAt": "2025-09-11T08:48:51.806Z", + "accessedAt": "2025-09-11T08:48:51.806Z", + "actId": null, + "actRunId": null, + "schema": null, + "stats": { + "readCount": 0, + "writeCount": 0, + "deleteCount": 0, + "listCount": 0, + "storageBytes": 0 + }, + "consoleUrl": "https://console.apify.com/storage/key-value-stores/someID", + "keysPublicUrl": "https://api.apify.com/v2/key-value-stores/someID/keys", + "generalAccess": "FOLLOW_USER_SETTING", + "urlSigningSecretKey": "urlSigningSecretKey" + } +}""" class TestKeyValueStoreSync: @@ -47,6 +73,19 @@ def test_key_value_store_should_create_public_keys_non_expiring_url(self, apify_ store.delete() assert apify_client.key_value_store(created_store['id']).get() is None + @parametrized_api_urls + def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: + apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url) + kvs = apify_client.key_value_store('someID') + + # Mock the API call to return predefined response + with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_KVS_RESPONSE)): + public_url = kvs.create_keys_public_url() + assert public_url == ( + f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/' + f'someID/keys?signature={public_url.split("signature=")[1]}' + ) + class TestKeyValueStoreAsync: async def test_key_value_store_should_create_expiring_keys_public_url_with_params( @@ -90,3 +129,16 @@ async def test_key_value_store_should_create_public_keys_non_expiring_url( await store.delete() assert await apify_client_async.key_value_store(created_store['id']).get() is None + + @parametrized_api_urls + async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None: + apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url) + kvs = apify_client.key_value_store('someID') + + # Mock the API call to return predefined response + with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_KVS_RESPONSE)): + public_url = await kvs.create_keys_public_url() + assert public_url == ( + f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/' + f'someID/keys?signature={public_url.split("signature=")[1]}' + )