Skip to content

feat: Add KeyValueStoreClient(Async).get_record_public_url #506

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Sep 30, 2025
Merged

feat: Add KeyValueStoreClient(Async).get_record_public_url #506

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
fbff42d
Update existing tests, add draft of the function
Pijukatel Sep 29, 2025
b999bc8
Finalize for kvs
Pijukatel Sep 29, 2025
7ac9e78
Fully parametrize new tests
Pijukatel Sep 29, 2025
1596855
Make mypy happy
Pijukatel Sep 29, 2025
06df3a2
Add signature value to the tests
Pijukatel Sep 30, 2025
e1bfec3
Make older versions of mypy happy
Pijukatel Sep 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 69 additions & 9 deletions src/apify_client/clients/resource_clients/key_value_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode, urlparse, urlunparse

from apify_shared.utils import create_storage_content_signature
from apify_shared.utils import create_hmac_signature, create_storage_content_signature

from apify_client._utils import (
catch_not_found_or_throw,
Expand Down Expand Up @@ -267,6 +267,36 @@ def delete_record(self, key: str) -> None:
timeout_secs=_SMALL_TIMEOUT,
)

def get_record_public_url(self, key: str) -> str:
"""Generate a URL that can be used to access key-value store record.

If the client has permission to access the key-value store's URL signing key, the URL will include a signature
to verify its authenticity.

Args:
key: The key for which the URL should be generated.

Returns:
A public URL that can be used to access the value of the given key in the KVS.
"""
if self.resource_id is None:
raise ValueError('resource_id cannot be None when generating a public URL')

metadata = self.get()

request_params = self._params()

if metadata and 'urlSigningSecretKey' in metadata:
request_params['signature'] = create_hmac_signature(metadata['urlSigningSecretKey'], key)

key_public_url = urlparse(self._url(f'records/{key}', public=True))
filtered_params = {k: v for k, v in request_params.items() if v is not None}

if filtered_params:
key_public_url = key_public_url._replace(query=urlencode(filtered_params))

return urlunparse(key_public_url)

def create_keys_public_url(
self,
*,
Expand All @@ -290,7 +320,7 @@ def create_keys_public_url(
Returns:
The public key-value store keys URL.
"""
store = self.get()
metadata = self.get()

request_params = self._params(
limit=limit,
Expand All @@ -299,10 +329,10 @@ def create_keys_public_url(
prefix=prefix,
)

if store and 'urlSigningSecretKey' in store:
if metadata and 'urlSigningSecretKey' in metadata:
signature = create_storage_content_signature(
resource_id=store['id'],
url_signing_secret_key=store['urlSigningSecretKey'],
resource_id=metadata['id'],
url_signing_secret_key=metadata['urlSigningSecretKey'],
expires_in_millis=expires_in_secs * 1000 if expires_in_secs is not None else None,
)
request_params['signature'] = signature
Expand Down Expand Up @@ -555,6 +585,36 @@ async def delete_record(self, key: str) -> None:
timeout_secs=_SMALL_TIMEOUT,
)

async def get_record_public_url(self, key: str) -> str:
"""Generate a URL that can be used to access key-value store record.

If the client has permission to access the key-value store's URL signing key, the URL will include a signature
to verify its authenticity.

Args:
key: The key for which the URL should be generated.

Returns:
A public URL that can be used to access the value of the given key in the KVS.
"""
if self.resource_id is None:
raise ValueError('resource_id cannot be None when generating a public URL')

metadata = await self.get()

request_params = self._params()

if metadata and 'urlSigningSecretKey' in metadata:
request_params['signature'] = create_hmac_signature(metadata['urlSigningSecretKey'], key)

key_public_url = urlparse(self._url(f'records/{key}', public=True))
filtered_params = {k: v for k, v in request_params.items() if v is not None}

if filtered_params:
key_public_url = key_public_url._replace(query=urlencode(filtered_params))

return urlunparse(key_public_url)

async def create_keys_public_url(
self,
*,
Expand All @@ -578,7 +638,7 @@ async def create_keys_public_url(
Returns:
The public key-value store keys URL.
"""
store = await self.get()
metadata = await self.get()

keys_public_url = urlparse(self._url('keys'))

Expand All @@ -589,10 +649,10 @@ async def create_keys_public_url(
prefix=prefix,
)

if store and 'urlSigningSecretKey' in store:
if metadata and 'urlSigningSecretKey' in metadata:
signature = create_storage_content_signature(
resource_id=store['id'],
url_signing_secret_key=store['urlSigningSecretKey'],
resource_id=metadata['id'],
url_signing_secret_key=metadata['urlSigningSecretKey'],
expires_in_millis=expires_in_secs * 1000 if expires_in_secs is not None else None,
)
request_params['signature'] = signature
Expand Down
113 changes: 81 additions & 32 deletions tests/integration/test_key_value_store.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,39 +1,40 @@
from __future__ import annotations

import json
from unittest import mock
from unittest.mock import Mock

import impit
import pytest

from integration.integration_test_utils import parametrized_api_urls, random_resource_name

from apify_client import ApifyClient, ApifyClientAsync
from apify_client.client import DEFAULT_API_URL

MOCKED_API_KVS_RESPONSE = """{
"data": {
"id": "someID",
"name": "name",
"userId": "userId",
"createdAt": "2025-09-11T08:48:51.806Z",
"modifiedAt": "2025-09-11T08:48:51.806Z",
"accessedAt": "2025-09-11T08:48:51.806Z",
"actId": null,
"actRunId": null,
"schema": null,
"stats": {
"readCount": 0,
"writeCount": 0,
"deleteCount": 0,
"listCount": 0,
"storageBytes": 0
},
"consoleUrl": "https://console.apify.com/storage/key-value-stores/someID",
"keysPublicUrl": "https://api.apify.com/v2/key-value-stores/someID/keys",
"generalAccess": "FOLLOW_USER_SETTING",
"urlSigningSecretKey": "urlSigningSecretKey"
}
}"""

def _get_mocked_api_kvs_response(signing_key: str | None = None) -> str:
response_data = {
'data': {
'id': 'someID',
'name': 'name',
'userId': 'userId',
'createdAt': '2025-09-11T08:48:51.806Z',
'modifiedAt': '2025-09-11T08:48:51.806Z',
'accessedAt': '2025-09-11T08:48:51.806Z',
'actId': None,
'actRunId': None,
'schema': None,
'stats': {'readCount': 0, 'writeCount': 0, 'deleteCount': 0, 'listCount': 0, 'storageBytes': 0},
'consoleUrl': 'https://console.apify.com/storage/key-value-stores/someID',
'keysPublicUrl': 'https://api.apify.com/v2/key-value-stores/someID/keys',
'generalAccess': 'FOLLOW_USER_SETTING',
}
}
if signing_key:
response_data['data']['urlSigningSecretKey'] = signing_key

return json.dumps(response_data)


class TestKeyValueStoreSync:
Expand Down Expand Up @@ -73,17 +74,41 @@ def test_key_value_store_should_create_public_keys_non_expiring_url(self, apify_
store.delete()
assert apify_client.key_value_store(created_store['id']).get() is None

@pytest.mark.parametrize('signing_key', [None, 'custom-signing-key'])
@parametrized_api_urls
def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
def test_public_url(self, api_token: str, api_url: str, api_public_url: str, signing_key: str) -> None:
apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url)
kvs = apify_client.key_value_store('someID')

# Mock the API call to return predefined response
with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_KVS_RESPONSE)):
with mock.patch.object(
apify_client.http_client,
'call',
return_value=Mock(text=_get_mocked_api_kvs_response(signing_key=signing_key)),
):
public_url = kvs.create_keys_public_url()
expected_signature = f'?signature={public_url.split("signature=")[1]}' if signing_key else ''
assert public_url == (
f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/someID/keys{expected_signature}'
)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In all the (new) tests, it seems we're parsing the signature query param from the public_url, only to test if public_url contains this query param.

something like

const { signature } = public_url;
assert public_url === `abc.def/${signature}`;

Can we check whether, e.g., the signature is the actual HMAC of the key instead?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added expected signature value to the tests


@pytest.mark.parametrize('signing_key', [None, 'custom-signing-key'])
@parametrized_api_urls
def test_record_public_url(self, api_token: str, api_url: str, api_public_url: str, signing_key: str) -> None:
apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url)
kvs = apify_client.key_value_store('someID')

# Mock the API call to return predefined response
with mock.patch.object(
apify_client.http_client,
'call',
return_value=Mock(text=_get_mocked_api_kvs_response(signing_key=signing_key)),
):
public_url = kvs.get_record_public_url(key='key')
expected_signature = f'?signature={public_url.split("signature=")[1]}' if signing_key else ''
assert public_url == (
f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/'
f'someID/keys?signature={public_url.split("signature=")[1]}'
f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/someID/'
f'records/key{expected_signature}'
)


Expand Down Expand Up @@ -130,15 +155,39 @@ async def test_key_value_store_should_create_public_keys_non_expiring_url(
await store.delete()
assert await apify_client_async.key_value_store(created_store['id']).get() is None

@pytest.mark.parametrize('signing_key', [None, 'custom-signing-key'])
@parametrized_api_urls
async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
async def test_record_public_url(self, api_token: str, api_url: str, api_public_url: str, signing_key: str) -> None:
apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url)
kvs = apify_client.key_value_store('someID')

# Mock the API call to return predefined response
with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_KVS_RESPONSE)):
with mock.patch.object(
apify_client.http_client,
'call',
return_value=Mock(text=_get_mocked_api_kvs_response(signing_key=signing_key)),
):
public_url = await kvs.create_keys_public_url()
expected_signature = f'?signature={public_url.split("signature=")[1]}' if signing_key else ''
assert public_url == (
f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/someID/keys{expected_signature}'
)

@pytest.mark.parametrize('signing_key', [None, 'custom-signing-key'])
@parametrized_api_urls
async def test_public_url(self, api_token: str, api_url: str, api_public_url: str, signing_key: str) -> None:
apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url)
kvs = apify_client.key_value_store('someID')

# Mock the API call to return predefined response
with mock.patch.object(
apify_client.http_client,
'call',
return_value=Mock(text=_get_mocked_api_kvs_response(signing_key=signing_key)),
):
public_url = await kvs.get_record_public_url(key='key')
expected_signature = f'?signature={public_url.split("signature=")[1]}' if signing_key else ''
assert public_url == (
f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/'
f'someID/keys?signature={public_url.split("signature=")[1]}'
f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/someID/'
f'records/key{expected_signature}'
)
Loading