fix: Presigned resource urls shouldn't follow base url

src/apify_client/client.py

@@ -64,6 +64,7 @@ def __init__(
         token: str | None = None,
         *,
         api_url: str | None = None,
+        api_public_url: str | None = None,
         max_retries: int | None = 8,
         min_delay_between_retries_millis: int | None = 500,
         timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -72,7 +73,10 @@ def __init__(
 
         Args:
             token: The Apify API token.
-            api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
+            api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can
+                be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well.
+            api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
+                is an internal URL that is not globally accessible.
             max_retries: How many times to retry a failed request at most.
             min_delay_between_retries_millis: How long will the client wait between retrying requests
                 (increases exponentially from this value).
@@ -81,6 +85,8 @@ def __init__(
         self.token = token
         api_url = (api_url or DEFAULT_API_URL).rstrip('/')
         self.base_url = f'{api_url}/{API_VERSION}'
+        api_public_url = (api_public_url or DEFAULT_API_URL).rstrip('/')
+        self.public_base_url = f'{api_public_url}/{API_VERSION}'
         self.max_retries = max_retries or 8
         self.min_delay_between_retries_millis = min_delay_between_retries_millis or 500
         self.timeout_secs = timeout_secs or DEFAULT_TIMEOUT
@@ -103,6 +109,7 @@ def __init__(
         token: str | None = None,
         *,
         api_url: str | None = None,
+        api_public_url: str | None = None,
         max_retries: int | None = 8,
         min_delay_between_retries_millis: int | None = 500,
         timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -111,7 +118,10 @@ def __init__(
 
         Args:
             token: The Apify API token.
-            api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
+            api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can
+                be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well.
+            api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
+                is an internal URL that is not globally accessible.
             max_retries: How many times to retry a failed request at most.
             min_delay_between_retries_millis: How long will the client wait between retrying requests
                 (increases exponentially from this value).
@@ -120,6 +130,7 @@ def __init__(
         super().__init__(
             token,
             api_url=api_url,
+            api_public_url=api_public_url,
             max_retries=max_retries,
             min_delay_between_retries_millis=min_delay_between_retries_millis,
             timeout_secs=timeout_secs,
@@ -286,6 +297,7 @@ def __init__(
         token: str | None = None,
         *,
         api_url: str | None = None,
+        api_public_url: str | None = None,
         max_retries: int | None = 8,
         min_delay_between_retries_millis: int | None = 500,
         timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -294,7 +306,10 @@ def __init__(
 
         Args:
             token: The Apify API token.
-            api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
+            api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can
+                be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well.
+            api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
+                is an internal URL that is not globally accessible.
             max_retries: How many times to retry a failed request at most.
             min_delay_between_retries_millis: How long will the client wait between retrying requests
                 (increases exponentially from this value).
@@ -303,6 +318,7 @@ def __init__(
         super().__init__(
             token,
             api_url=api_url,
+            api_public_url=api_public_url,
             max_retries=max_retries,
             min_delay_between_retries_millis=min_delay_between_retries_millis,
             timeout_secs=timeout_secs,

src/apify_client/clients/base/base_client.py

@@ -18,10 +18,12 @@ class _BaseBaseClient(metaclass=WithLogDetailsClient):
     http_client: HTTPClient | HTTPClientAsync
     root_client: ApifyClient | ApifyClientAsync
 
-    def _url(self, path: str | None = None) -> str:
-        if path is not None:
-            return f'{self.url}/{path}'
-        return self.url
+    def _url(self, path: str | None = None, *, public: bool = False) -> str:
+        url = f'{self.url}/{path}' if path is not None else self.url
+
+        if public:
+            return url.replace(self.root_client.base_url, self.root_client.public_base_url)
+        return url
 
     def _params(self, **kwargs: Any) -> dict:
         return {

src/apify_client/clients/resource_clients/dataset.py

@@ -619,7 +619,7 @@ def create_items_public_url(
             )
             request_params['signature'] = signature
 
-        items_public_url = urlparse(self._url('items'))
+        items_public_url = urlparse(self._url('items', public=True))
         filtered_params = {k: v for k, v in request_params.items() if v is not None}
         if filtered_params:
             items_public_url = items_public_url._replace(query=urlencode(filtered_params))
@@ -1126,7 +1126,7 @@ async def create_items_public_url(
             )
             request_params['signature'] = signature
 
-        items_public_url = urlparse(self._url('items'))
+        items_public_url = urlparse(self._url('items', public=True))
         filtered_params = {k: v for k, v in request_params.items() if v is not None}
         if filtered_params:
             items_public_url = items_public_url._replace(query=urlencode(filtered_params))

src/apify_client/clients/resource_clients/key_value_store.py

@@ -307,7 +307,7 @@ def create_keys_public_url(
             )
             request_params['signature'] = signature
 
-        keys_public_url = urlparse(self._url('keys'))
+        keys_public_url = urlparse(self._url('keys', public=True))
 
         filtered_params = {k: v for k, v in request_params.items() if v is not None}
         if filtered_params:
@@ -597,7 +597,7 @@ async def create_keys_public_url(
             )
             request_params['signature'] = signature
 
-        keys_public_url = urlparse(self._url('keys'))
+        keys_public_url = urlparse(self._url('keys', public=True))
         filtered_params = {k: v for k, v in request_params.items() if v is not None}
         if filtered_params:
             keys_public_url = keys_public_url._replace(query=urlencode(filtered_params))

tests/integration/conftest.py

@@ -7,15 +7,29 @@
 TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN'
 API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL'
 
+parametrized_api_urls = pytest.mark.parametrize(
+    ('api_url', 'api_public_url'),
+    [
+        ('https://api.apify.com', 'https://api.apify.com'),
+        ('https://api.apify.com', None),
+        ('https://api.apify.com', 'https://custom-public-url.com'),
+        ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path'),
+        ('https://api.apify.com', 'https://custom-public-url.com/with/custom/path/'),
+    ],
+)
 
-@pytest.fixture
-def apify_client() -> ApifyClient:
-    api_token = os.getenv(TOKEN_ENV_VAR)
-    api_url = os.getenv(API_URL_ENV_VAR)
 
-    if not api_token:
+@pytest.fixture
+def api_token() -> str:
+    token = os.getenv(TOKEN_ENV_VAR)
+    if not token:
         raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
+    return token
 
+
+@pytest.fixture
+def apify_client(api_token: str) -> ApifyClient:
+    api_url = os.getenv(API_URL_ENV_VAR)
     return ApifyClient(api_token, api_url=api_url)
 
 
@@ -25,11 +39,6 @@ def apify_client() -> ApifyClient:
 # but `pytest-asyncio` closes the event loop after each test,
 # and uses a new one for the next test.
 @pytest.fixture
-def apify_client_async() -> ApifyClientAsync:
-    api_token = os.getenv(TOKEN_ENV_VAR)
+def apify_client_async(api_token: str) -> ApifyClientAsync:
     api_url = os.getenv(API_URL_ENV_VAR)
-
-    if not api_token:
-        raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
-
     return ApifyClientAsync(api_token, api_url=api_url)

tests/integration/test_dataset.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
 import impit
 
+from integration.conftest import parametrized_api_urls
 from integration.integration_test_utils import random_resource_name
 
-if TYPE_CHECKING:
-    from apify_client import ApifyClient, ApifyClientAsync
+from apify_client import ApifyClient, ApifyClientAsync
+from apify_client.client import DEFAULT_API_URL
 
 
 class TestDatasetSync:
@@ -47,6 +46,28 @@ def test_dataset_should_create_public_items_non_expiring_url(self, apify_client:
         dataset.delete()
         assert apify_client.dataset(created_dataset['id']).get() is None
 
+    @parametrized_api_urls
+    def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
+        apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url)
+        created_store = apify_client.datasets().get_or_create(name=random_resource_name('key-value-store'))
+        dataset = apify_client.dataset(created_store['id'])
+        try:
+            public_url = dataset.create_items_public_url()
+            assert public_url == (
+                f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/'
+                f'{created_store["id"]}/items?signature={public_url.split("signature=")[1]}'
+            )
+        finally:
+            dataset.delete()
+
+    def test_public_url_nonexistent_host(self, api_token: str) -> None:
+        dataset_name = 'whatever'
+        non_existent_url = 'http://10.0.88.214:8010'
+        apify_client = ApifyClient(token=api_token, api_url=non_existent_url)
+        kvs_client = apify_client.dataset(dataset_id=dataset_name)
+        assert kvs_client._url() == f'{non_existent_url}/v2/datasets/{dataset_name}'
+        assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/datasets/{dataset_name}'
+
 
 class TestDatasetAsync:
     async def test_dataset_should_create_public_items_expiring_url_with_params(
@@ -88,3 +109,25 @@ async def test_dataset_should_create_public_items_non_expiring_url(
 
         await dataset.delete()
         assert await apify_client_async.dataset(created_dataset['id']).get() is None
+
+    @parametrized_api_urls
+    async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
+        apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url)
+        created_store = await apify_client.datasets().get_or_create(name=random_resource_name('key-value-store'))
+        dataset = apify_client.dataset(created_store['id'])
+        try:
+            public_url = await dataset.create_items_public_url()
+            assert public_url == (
+                f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/'
+                f'{created_store["id"]}/items?signature={public_url.split("signature=")[1]}'
+            )
+        finally:
+            await dataset.delete()
+
+    def test_public_url_nonexistent_host(self, api_token: str) -> None:
+        dataset_name = 'whatever'
+        non_existent_url = 'http://10.0.88.214:8010'
+        apify_client = ApifyClientAsync(token=api_token, api_url=non_existent_url)
+        kvs_client = apify_client.dataset(dataset_id=dataset_name)
+        assert kvs_client._url() == f'{non_existent_url}/v2/datasets/{dataset_name}'
+        assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/datasets/{dataset_name}'

tests/integration/test_key_value_store.py

@@ -1,13 +1,12 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
 import impit
 
+from integration.conftest import parametrized_api_urls
 from integration.integration_test_utils import random_resource_name
 
-if TYPE_CHECKING:
-    from apify_client import ApifyClient, ApifyClientAsync
+from apify_client import ApifyClient, ApifyClientAsync
+from apify_client.client import DEFAULT_API_URL
 
 
 class TestKeyValueStoreSync:
@@ -47,6 +46,28 @@ def test_key_value_store_should_create_public_keys_non_expiring_url(self, apify_
         store.delete()
         assert apify_client.key_value_store(created_store['id']).get() is None
 
+    @parametrized_api_urls
+    def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
+        apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url)
+        created_store = apify_client.key_value_stores().get_or_create(name=random_resource_name('key-value-store'))
+        kvs = apify_client.key_value_store(created_store['id'])
+        try:
+            public_url = kvs.create_keys_public_url()
+            assert public_url == (
+                f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/'
+                f'{created_store["id"]}/keys?signature={public_url.split("signature=")[1]}'
+            )
+        finally:
+            kvs.delete()
+
+    def test_public_url_nonexistent_host(self, api_token: str) -> None:
+        kvs_name = 'whatever'
+        non_existent_url = 'http://10.0.88.214:8010'
+        apify_client = ApifyClient(token=api_token, api_url=non_existent_url)
+        kvs_client = apify_client.key_value_store(key_value_store_id=kvs_name)
+        assert kvs_client._url() == f'{non_existent_url}/v2/key-value-stores/{kvs_name}'
+        assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/key-value-stores/{kvs_name}'
+
 
 class TestKeyValueStoreAsync:
     async def test_key_value_store_should_create_expiring_keys_public_url_with_params(
@@ -90,3 +111,27 @@ async def test_key_value_store_should_create_public_keys_non_expiring_url(
 
         await store.delete()
         assert await apify_client_async.key_value_store(created_store['id']).get() is None
+
+    @parametrized_api_urls
+    async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
+        apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url)
+        created_store = await apify_client.key_value_stores().get_or_create(
+            name=random_resource_name('key-value-store')
+        )
+        kvs = apify_client.key_value_store(created_store['id'])
+        try:
+            public_url = await kvs.create_keys_public_url()
+            assert public_url == (
+                f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/'
+                f'{created_store["id"]}/keys?signature={public_url.split("signature=")[1]}'
+            )
+        finally:
+            await kvs.delete()
+
+    async def test_public_url_nonexistent_host(self, api_token: str) -> None:
+        kvs_name = 'whatever'
+        non_existent_url = 'http://10.0.88.214:8010'
+        apify_client = ApifyClientAsync(token=api_token, api_url=non_existent_url)
+        kvs_client = apify_client.key_value_store(key_value_store_id=kvs_name)
+        assert kvs_client._url() == f'{non_existent_url}/v2/key-value-stores/{kvs_name}'
+        assert kvs_client._url(public=True) == f'{DEFAULT_API_URL}/v2/key-value-stores/{kvs_name}'