fix: Presigned resource urls shouldn't follow base url (#500)

### Description

- Allow the possibility to define a custom public URL; if not used, the
default API public URL will be used.
- Python version of the following fix JS client fix
apify/apify-client-js#745

### Issues

- Closes: #496

### Testing

- Added new tests for relevant kvs and datasets clients' methods.

---------

Co-authored-by: Jindřich Bär <[email protected]>

Author: Pijukatel

src/apify_client/client.py

@@ -64,6 +64,7 @@ def __init__(
         token: str | None = None,
         *,
         api_url: str | None = None,
+        api_public_url: str | None = None,
         max_retries: int | None = 8,
         min_delay_between_retries_millis: int | None = 500,
         timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -72,7 +73,10 @@ def __init__(
 
         Args:
             token: The Apify API token.
-            api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
+            api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can
+                be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well.
+            api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
+                is an internal URL that is not globally accessible.
             max_retries: How many times to retry a failed request at most.
             min_delay_between_retries_millis: How long will the client wait between retrying requests
                 (increases exponentially from this value).
@@ -81,6 +85,8 @@ def __init__(
         self.token = token
         api_url = (api_url or DEFAULT_API_URL).rstrip('/')
         self.base_url = f'{api_url}/{API_VERSION}'
+        api_public_url = (api_public_url or DEFAULT_API_URL).rstrip('/')
+        self.public_base_url = f'{api_public_url}/{API_VERSION}'
         self.max_retries = max_retries or 8
         self.min_delay_between_retries_millis = min_delay_between_retries_millis or 500
         self.timeout_secs = timeout_secs or DEFAULT_TIMEOUT
@@ -103,6 +109,7 @@ def __init__(
         token: str | None = None,
         *,
         api_url: str | None = None,
+        api_public_url: str | None = None,
         max_retries: int | None = 8,
         min_delay_between_retries_millis: int | None = 500,
         timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -111,7 +118,10 @@ def __init__(
 
         Args:
             token: The Apify API token.
-            api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
+            api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can
+                be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well.
+            api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
+                is an internal URL that is not globally accessible.
             max_retries: How many times to retry a failed request at most.
             min_delay_between_retries_millis: How long will the client wait between retrying requests
                 (increases exponentially from this value).
@@ -120,6 +130,7 @@ def __init__(
         super().__init__(
             token,
             api_url=api_url,
+            api_public_url=api_public_url,
             max_retries=max_retries,
             min_delay_between_retries_millis=min_delay_between_retries_millis,
             timeout_secs=timeout_secs,
@@ -286,6 +297,7 @@ def __init__(
         token: str | None = None,
         *,
         api_url: str | None = None,
+        api_public_url: str | None = None,
         max_retries: int | None = 8,
         min_delay_between_retries_millis: int | None = 500,
         timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -294,7 +306,10 @@ def __init__(
 
         Args:
             token: The Apify API token.
-            api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
+            api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can
+                be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well.
+            api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
+                is an internal URL that is not globally accessible.
             max_retries: How many times to retry a failed request at most.
             min_delay_between_retries_millis: How long will the client wait between retrying requests
                 (increases exponentially from this value).
@@ -303,6 +318,7 @@ def __init__(
         super().__init__(
             token,
             api_url=api_url,
+            api_public_url=api_public_url,
             max_retries=max_retries,
             min_delay_between_retries_millis=min_delay_between_retries_millis,
             timeout_secs=timeout_secs,

src/apify_client/clients/base/base_client.py

@@ -18,10 +18,14 @@ class _BaseBaseClient(metaclass=WithLogDetailsClient):
     http_client: HTTPClient | HTTPClientAsync
     root_client: ApifyClient | ApifyClientAsync
 
-    def _url(self, path: str | None = None) -> str:
-        if path is not None:
-            return f'{self.url}/{path}'
-        return self.url
+    def _url(self, path: str | None = None, *, public: bool = False) -> str:
+        url = f'{self.url}/{path}' if path is not None else self.url
+
+        if public:
+            if not url.startswith(self.root_client.base_url):
+                raise ValueError('API based URL has to start with `self.root_client.base_url`')
+            return url.replace(self.root_client.base_url, self.root_client.public_base_url, 1)
+        return url
 
     def _params(self, **kwargs: Any) -> dict:
         return {

src/apify_client/clients/resource_clients/dataset.py

@@ -619,7 +619,7 @@ def create_items_public_url(
             )
             request_params['signature'] = signature
 
-        items_public_url = urlparse(self._url('items'))
+        items_public_url = urlparse(self._url('items', public=True))
         filtered_params = {k: v for k, v in request_params.items() if v is not None}
         if filtered_params:
             items_public_url = items_public_url._replace(query=urlencode(filtered_params))
@@ -1126,7 +1126,7 @@ async def create_items_public_url(
             )
             request_params['signature'] = signature
 
-        items_public_url = urlparse(self._url('items'))
+        items_public_url = urlparse(self._url('items', public=True))
         filtered_params = {k: v for k, v in request_params.items() if v is not None}
         if filtered_params:
             items_public_url = items_public_url._replace(query=urlencode(filtered_params))

src/apify_client/clients/resource_clients/key_value_store.py

@@ -307,7 +307,7 @@ def create_keys_public_url(
             )
             request_params['signature'] = signature
 
-        keys_public_url = urlparse(self._url('keys'))
+        keys_public_url = urlparse(self._url('keys', public=True))
 
         filtered_params = {k: v for k, v in request_params.items() if v is not None}
         if filtered_params:
@@ -597,7 +597,7 @@ async def create_keys_public_url(
             )
             request_params['signature'] = signature
 
-        keys_public_url = urlparse(self._url('keys'))
+        keys_public_url = urlparse(self._url('keys', public=True))
         filtered_params = {k: v for k, v in request_params.items() if v is not None}
         if filtered_params:
             keys_public_url = keys_public_url._replace(query=urlencode(filtered_params))

tests/integration/conftest.py

@@ -9,13 +9,16 @@
 
 
 @pytest.fixture
-def apify_client() -> ApifyClient:
-    api_token = os.getenv(TOKEN_ENV_VAR)
-    api_url = os.getenv(API_URL_ENV_VAR)
-
-    if not api_token:
+def api_token() -> str:
+    token = os.getenv(TOKEN_ENV_VAR)
+    if not token:
         raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
+    return token
 
+
+@pytest.fixture
+def apify_client(api_token: str) -> ApifyClient:
+    api_url = os.getenv(API_URL_ENV_VAR)
     return ApifyClient(api_token, api_url=api_url)
 
 
@@ -25,11 +28,6 @@ def apify_client() -> ApifyClient:
 # but `pytest-asyncio` closes the event loop after each test,
 # and uses a new one for the next test.
 @pytest.fixture
-def apify_client_async() -> ApifyClientAsync:
-    api_token = os.getenv(TOKEN_ENV_VAR)
+def apify_client_async(api_token: str) -> ApifyClientAsync:
     api_url = os.getenv(API_URL_ENV_VAR)
-
-    if not api_token:
-        raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
-
     return ApifyClientAsync(api_token, api_url=api_url)

tests/integration/integration_test_utils.py

@@ -1,10 +1,26 @@
 import secrets
 import string
 
+import pytest
+
 
 def random_string(length: int = 10) -> str:
     return ''.join(secrets.choice(string.ascii_letters) for _ in range(length))
 
 
 def random_resource_name(resource: str) -> str:
     return f'python-client-test-{resource}-{random_string(5)}'
+
+
+parametrized_api_urls = pytest.mark.parametrize(
+    ('api_url', 'api_public_url'),
+    [
+        ('https://api.apify.com', 'https://api.apify.com'),
+        ('https://api.apify.com', None),
+        ('https://api.apify.com', 'https://custom-public-url.com'),
+        ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path'),
+        ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path/'),
+        ('http://10.0.88.214:8010', 'https://api.apify.com'),
+        ('http://10.0.88.214:8010', None),
+    ],
+)

tests/integration/test_dataset.py

@@ -1,13 +1,40 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from unittest import mock
+from unittest.mock import Mock
 
 import impit
 
-from integration.integration_test_utils import random_resource_name
-
-if TYPE_CHECKING:
-    from apify_client import ApifyClient, ApifyClientAsync
+from integration.integration_test_utils import parametrized_api_urls, random_resource_name
+
+from apify_client import ApifyClient, ApifyClientAsync
+from apify_client.client import DEFAULT_API_URL
+
+MOCKED_API_DATASET_RESPONSE = """{
+  "data": {
+    "id": "someID",
+    "name": "name",
+    "userId": "userId",
+    "createdAt": "2025-09-11T08:48:51.806Z",
+    "modifiedAt": "2025-09-11T08:48:51.806Z",
+    "accessedAt": "2025-09-11T08:48:51.806Z",
+    "actId": null,
+    "actRunId": null,
+    "schema": null,
+    "stats": {
+      "readCount": 0,
+      "writeCount": 0,
+      "deleteCount": 0,
+      "listCount": 0,
+      "storageBytes": 0
+    },
+    "fields": [],
+    "consoleUrl": "https://console.apify.com/storage/datasets/someID",
+    "itemsPublicUrl": "https://api.apify.com/v2/datasets/someID/items",
+    "generalAccess": "FOLLOW_USER_SETTING",
+    "urlSigningSecretKey": "urlSigningSecretKey"
+  }
+}"""
 
 
 class TestDatasetSync:
@@ -47,6 +74,19 @@ def test_dataset_should_create_public_items_non_expiring_url(self, apify_client:
         dataset.delete()
         assert apify_client.dataset(created_dataset['id']).get() is None
 
+    @parametrized_api_urls
+    def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
+        apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url)
+        dataset = apify_client.dataset('someID')
+
+        # Mock the API call to return predefined response
+        with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_DATASET_RESPONSE)):
+            public_url = dataset.create_items_public_url()
+            assert public_url == (
+                f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/'
+                f'someID/items?signature={public_url.split("signature=")[1]}'
+            )
+
 
 class TestDatasetAsync:
     async def test_dataset_should_create_public_items_expiring_url_with_params(
@@ -88,3 +128,16 @@ async def test_dataset_should_create_public_items_non_expiring_url(
 
         await dataset.delete()
         assert await apify_client_async.dataset(created_dataset['id']).get() is None
+
+    @parametrized_api_urls
+    async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
+        apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url)
+        dataset = apify_client.dataset('someID')
+
+        # Mock the API call to return predefined response
+        with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_DATASET_RESPONSE)):
+            public_url = await dataset.create_items_public_url()
+            assert public_url == (
+                f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/'
+                f'someID/items?signature={public_url.split("signature=")[1]}'
+            )

tests/integration/test_key_value_store.py

@@ -1,13 +1,39 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from unittest import mock
+from unittest.mock import Mock
 
 import impit
 
-from integration.integration_test_utils import random_resource_name
-
-if TYPE_CHECKING:
-    from apify_client import ApifyClient, ApifyClientAsync
+from integration.integration_test_utils import parametrized_api_urls, random_resource_name
+
+from apify_client import ApifyClient, ApifyClientAsync
+from apify_client.client import DEFAULT_API_URL
+
+MOCKED_API_KVS_RESPONSE = """{
+  "data": {
+    "id": "someID",
+    "name": "name",
+    "userId": "userId",
+    "createdAt": "2025-09-11T08:48:51.806Z",
+    "modifiedAt": "2025-09-11T08:48:51.806Z",
+    "accessedAt": "2025-09-11T08:48:51.806Z",
+    "actId": null,
+    "actRunId": null,
+    "schema": null,
+    "stats": {
+      "readCount": 0,
+      "writeCount": 0,
+      "deleteCount": 0,
+      "listCount": 0,
+      "storageBytes": 0
+    },
+    "consoleUrl": "https://console.apify.com/storage/key-value-stores/someID",
+    "keysPublicUrl": "https://api.apify.com/v2/key-value-stores/someID/keys",
+    "generalAccess": "FOLLOW_USER_SETTING",
+    "urlSigningSecretKey": "urlSigningSecretKey"
+  }
+}"""
 
 
 class TestKeyValueStoreSync:
@@ -47,6 +73,19 @@ def test_key_value_store_should_create_public_keys_non_expiring_url(self, apify_
         store.delete()
         assert apify_client.key_value_store(created_store['id']).get() is None
 
+    @parametrized_api_urls
+    def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
+        apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url)
+        kvs = apify_client.key_value_store('someID')
+
+        # Mock the API call to return predefined response
+        with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_KVS_RESPONSE)):
+            public_url = kvs.create_keys_public_url()
+            assert public_url == (
+                f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/'
+                f'someID/keys?signature={public_url.split("signature=")[1]}'
+            )
+
 
 class TestKeyValueStoreAsync:
     async def test_key_value_store_should_create_expiring_keys_public_url_with_params(
@@ -90,3 +129,16 @@ async def test_key_value_store_should_create_public_keys_non_expiring_url(
 
         await store.delete()
         assert await apify_client_async.key_value_store(created_store['id']).get() is None
+
+    @parametrized_api_urls
+    async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
+        apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url)
+        kvs = apify_client.key_value_store('someID')
+
+        # Mock the API call to return predefined response
+        with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_KVS_RESPONSE)):
+            public_url = await kvs.create_keys_public_url()
+            assert public_url == (
+                f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/'
+                f'someID/keys?signature={public_url.split("signature=")[1]}'
+            )