feat: add new methods Dataset.createItemsPublicUrl & KeyValueStore.createKeysPublicUrl (#720)

When storage resources (Datasets or Key-Value Stores) are set to
Restricted, accessing or sharing their data externally becomes difficult
due to limited permissions. This PR introduces functionality to generate
signed URLs that allow controlled external access to these resources
without adding token to the request.

This PR introduces methods to generate signed URLs for Dataset items and
Key-Value Store records:
1. **Datasets**
`dataset(:datasetId).createItemsPublicUrl(options, expiresInMillis)`
→ Returns a signed URL like:
`/v2/datasets/:datasetId/items?signature=xxx`

2. Key-Value Stores
`keyValueStore(:storeId).createKeysPublicUrl(options, expiresInMillis)`
→ Returns a signed URL like:
`/v2/key-value-stores/:storeId/keys?signature=xxx`

🕒 Expiration:
The `expiresInMillis` parameter defines how long the signature is valid.
- If provided, the URL will expire after the specified time.
- If omitted, the URL will never expire.


Note: The signature is included only if the token has WRITE access to
the storage. Otherwise, an unsigned URL is returned.

P.S. We're not yet exposing `urlSigningSecretKey` for datasets, it will
be released after [PR](apify/apify-core#22173)
is merged.

[More context
here](https://www.notion.so/apify/Signed-Dataset-Items-KV-store-record-URLs-224f39950a2280158a6bd82bc2e2ebb5?source=copy_link)

Author: danpoletaev

package-lock.json

@@ -64,6 +64,7 @@
     "dependencies": {
         "@apify/consts": "^2.25.0",
         "@apify/log": "^2.2.6",
+        "@apify/utilities": "^2.18.0",
         "@crawlee/types": "^3.3.0",
         "agentkeepalive": "^4.2.1",
         "async-retry": "^1.3.3",

package.json

@@ -1,6 +1,7 @@
 import ow from 'ow';
 
 import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';
+import { createStorageContentSignature } from '@apify/utilities';
 
 import type { ApifyApiError } from '../apify_api_error';
 import type { ApiClientSubResourceOptions } from '../base/api_client';
@@ -12,7 +13,7 @@ import {
 } from '../base/resource_client';
 import type { ApifyRequestConfig, ApifyResponse } from '../http_client';
 import type { PaginatedList } from '../utils';
-import { cast, catchNotFoundOrThrow, pluckData } from '../utils';
+import { applyQueryParamsToUrl, cast, catchNotFoundOrThrow, pluckData } from '../utils';
 
 export class DatasetClient<
     Data extends Record<string | number, any> = Record<string | number, unknown>,
@@ -163,6 +164,54 @@ export class DatasetClient<
         return undefined;
     }
 
+    /**
+     * Generates a URL that can be used to access dataset items.
+     *
+     * If the client has permission to access the dataset's URL signing key,
+     * the URL will include a signature to verify its authenticity.
+     *
+     * You can optionally control how long the signed URL should be valid using the `expiresInMillis` option.
+     * This value sets the expiration duration in milliseconds from the time the URL is generated.
+     * If not provided, the URL will not expire.
+     *
+     * Any other options (like `limit` or `prefix`) will be included as query parameters in the URL.
+     */
+    async createItemsPublicUrl(options: DatasetClientListItemOptions = {}, expiresInMillis?: number): Promise<string> {
+        ow(
+            options,
+            ow.object.exactShape({
+                clean: ow.optional.boolean,
+                desc: ow.optional.boolean,
+                flatten: ow.optional.array.ofType(ow.string),
+                fields: ow.optional.array.ofType(ow.string),
+                omit: ow.optional.array.ofType(ow.string),
+                limit: ow.optional.number,
+                offset: ow.optional.number,
+                skipEmpty: ow.optional.boolean,
+                skipHidden: ow.optional.boolean,
+                unwind: ow.optional.any(ow.string, ow.array.ofType(ow.string)),
+                view: ow.optional.string,
+            }),
+        );
+
+        const dataset = await this.get();
+
+        let createdItemsPublicUrl = new URL(this._url('items'));
+
+        if (dataset?.urlSigningSecretKey) {
+            const signature = createStorageContentSignature({
+                resourceId: dataset.id,
+                urlSigningSecretKey: dataset.urlSigningSecretKey,
+                expiresInMillis,
+            });
+            createdItemsPublicUrl.searchParams.set('signature', signature);
+        }
+
+        createdItemsPublicUrl = applyQueryParamsToUrl(createdItemsPublicUrl, options);
+
+        return createdItemsPublicUrl.toString();
+    }
+
     private _createPaginationList(response: ApifyResponse, userProvidedDesc: boolean): PaginatedList<Data> {
         return {
             items: response.data,
@@ -191,6 +240,8 @@ export interface Dataset {
     stats: DatasetStats;
     fields: string[];
     generalAccess?: STORAGE_GENERAL_ACCESS | null;
+    urlSigningSecretKey?: string | null;
+    itemsPublicUrl: string;
 }
 
 export interface DatasetStats {

src/resource_clients/dataset.ts

@@ -5,6 +5,7 @@ import type { JsonValue } from 'type-fest';
 
 import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';
 import log from '@apify/log';
+import { createStorageContentSignature } from '@apify/utilities';
 
 import type { ApifyApiError } from '../apify_api_error';
 import type { ApiClientSubResourceOptions } from '../base/api_client';
@@ -15,7 +16,16 @@ import {
     SMALL_TIMEOUT_MILLIS,
 } from '../base/resource_client';
 import type { ApifyRequestConfig } from '../http_client';
-import { cast, catchNotFoundOrThrow, isBuffer, isNode, isStream, parseDateFields, pluckData } from '../utils';
+import {
+    applyQueryParamsToUrl,
+    cast,
+    catchNotFoundOrThrow,
+    isBuffer,
+    isNode,
+    isStream,
+    parseDateFields,
+    pluckData,
+} from '../utils';
 
 export class KeyValueStoreClient extends ResourceClient {
     /**
@@ -75,6 +85,48 @@ export class KeyValueStoreClient extends ResourceClient {
         return cast(parseDateFields(pluckData(response.data)));
     }
 
+    /**
+     * Generates a URL that can be used to access key-value store keys.
+     *
+     * If the client has permission to access the key-value store's URL signing key,
+     * the URL will include a signature to verify its authenticity.
+     *
+     * You can optionally control how long the signed URL should be valid using the `expiresInMillis` option.
+     * This value sets the expiration duration in milliseconds from the time the URL is generated.
+     * If not provided, the URL will not expire.
+     *
+     * Any other options (like `limit` or `prefix`) will be included as query parameters in the URL.
+     *
+     */
+    async createKeysPublicUrl(options: KeyValueClientListKeysOptions = {}, expiresInMillis?: number) {
+        ow(
+            options,
+            ow.object.exactShape({
+                limit: ow.optional.number,
+                exclusiveStartKey: ow.optional.string,
+                collection: ow.optional.string,
+                prefix: ow.optional.string,
+            }),
+        );
+
+        const store = await this.get();
+
+        let createdPublicKeysUrl = new URL(this._url('items'));
+
+        if (store?.urlSigningSecretKey) {
+            const signature = createStorageContentSignature({
+                resourceId: store.id,
+                urlSigningSecretKey: store.urlSigningSecretKey,
+                expiresInMillis,
+            });
+            createdPublicKeysUrl.searchParams.set('signature', signature);
+        }
+
+        createdPublicKeysUrl = applyQueryParamsToUrl(createdPublicKeysUrl, options);
+
+        return createdPublicKeysUrl.toString();
+    }
+
     /**
      * Tests whether a record with the given key exists in the key-value store without retrieving its value.
      *
@@ -255,6 +307,8 @@ export interface KeyValueStore {
     actRunId?: string;
     stats?: KeyValueStoreStats;
     generalAccess?: STORAGE_GENERAL_ACCESS | null;
+    urlSigningSecretKey?: string | null;
+    keysPublicUrl: string;
 }
 
 export interface KeyValueStoreStats {

src/resource_clients/key_value_store.ts

@@ -260,3 +260,23 @@ export function asArray<T>(value: T | T[]): T[] {
 export type Dictionary<T = unknown> = Record<PropertyKey, T>;
 
 export type DistributiveOptional<T, K extends keyof T> = T extends any ? Omit<T, K> & Partial<Pick<T, K>> : never;
+
+/**
+ * Adds query parameters to a given URL based on the provided options object.
+ */
+export function applyQueryParamsToUrl(
+    url: URL,
+    options?: Record<string, string | number | boolean | string[] | undefined>,
+) {
+    for (const [key, value] of Object.entries(options ?? {})) {
+        // skip undefined values
+        if (value === undefined) continue;
+        // join array values with a comma
+        if (Array.isArray(value)) {
+            url.searchParams.set(key, value.join(','));
+            continue;
+        }
+        url.searchParams.set(key, String(value));
+    }
+    return url;
+}

src/utils.ts

@@ -327,5 +327,32 @@ describe('Dataset methods', () => {
             expect(browserRes).toEqual(res);
             validateRequest({}, { datasetId });
         });
+
+        describe('createItemsPublicUrl()', () => {
+            it('should include a signature in the URL when the caller has permission to access the signing secret key', async () => {
+                const datasetId = 'id-with-secret-key';
+                const res = await client.dataset(datasetId).createItemsPublicUrl();
+
+                expect(new URL(res).searchParams.get('signature')).toBeDefined();
+            });
+
+            it('should not include a signature in the URL when the caller lacks permission to access the signing secret key', async () => {
+                const datasetId = 'some-id';
+                const res = await client.dataset(datasetId).createItemsPublicUrl();
+
+                expect(new URL(res).searchParams.get('signature')).toBeNull();
+            });
+
+            it('includes provided options (e.g., limit and prefix) as query parameters', async () => {
+                const datasetId = 'id-with-secret-key';
+                const res = await client.dataset(datasetId).createItemsPublicUrl({ desc: true, limit: 10, offset: 5 });
+                const itemsPublicUrl = new URL(res);
+
+                expect(itemsPublicUrl.searchParams.get('desc')).toBe('true');
+                expect(itemsPublicUrl.searchParams.get('limit')).toBe('10');
+                expect(itemsPublicUrl.searchParams.get('offset')).toBe('5');
+                expect(itemsPublicUrl.searchParams.get('signature')).toBeDefined();
+            });
+        });
     });
 });

test/datasets.test.js

@@ -553,5 +553,31 @@ describe('Key-Value Store methods', () => {
             expect(browserRes).toBeUndefined();
             validateRequest({}, { storeId, key });
         });
+
+        describe('createKeysPublicUrl()', () => {
+            it('should include a signature in the URL when the caller has permission to access the signing secret key', async () => {
+                const storeId = 'id-with-secret-key';
+                const res = await client.keyValueStore(storeId).createKeysPublicUrl();
+
+                expect(new URL(res).searchParams.get('signature')).toBeDefined();
+            });
+
+            it('should not include a signature in the URL when the caller lacks permission to access the signing secret key', async () => {
+                const storeId = 'some-id';
+                const res = await client.keyValueStore(storeId).createKeysPublicUrl();
+
+                expect(new URL(res).searchParams.get('signature')).toBeNull();
+            });
+
+            it('includes provided options (e.g., limit and prefix) as query parameters', async () => {
+                const storeId = 'id-with-secret-key';
+                const res = await client.keyValueStore(storeId).createKeysPublicUrl({ limit: 10, prefix: 'prefix' });
+                const keysPublicUrl = new URL(res);
+
+                expect(keysPublicUrl.searchParams.get('limit')).toBe('10');
+                expect(keysPublicUrl.searchParams.get('prefix')).toBe('prefix');
+                expect(keysPublicUrl.searchParams.get('signature')).toBeDefined();
+            });
+        });
     });
 });

test/key_value_stores.test.js

@@ -7,7 +7,6 @@ const datasets = express.Router();
 const ROUTES = [
     { id: 'get-or-create-dataset', method: 'POST', path: '/' },
     { id: 'list-datasets', method: 'GET', path: '/' },
-    { id: 'get-dataset', method: 'GET', path: '/:datasetId' },
     { id: 'delete-dataset', method: 'DELETE', path: '/:datasetId' },
     { id: 'update-dataset', method: 'PUT', path: '/:datasetId' },
     { id: 'list-items', method: 'GET', path: '/:datasetId/items', type: 'responseJsonMock' },
@@ -17,4 +16,34 @@ const ROUTES = [
 
 addRoutes(datasets, ROUTES);
 
+/**
+ * GET /datasets/:datasetId
+ * Returns a specific dataset by its ID.
+ * If the dataset ID is 'id-with-secret-key', it returns a dataset with a URL signing secret key.
+ * If the dataset ID is '404', it returns a 404 error with a RECORD_NOT_FOUND type.
+ * Otherwise, it returns a dataset with an ID of 'get-dataset' (default).
+ */
+datasets.get('/:datasetId', (req, res) => {
+    const { datasetId } = req.params;
+
+    if (datasetId === 'id-with-secret-key') {
+        return res.json({
+            data: {
+                id: datasetId,
+                urlSigningSecretKey: 'secret-key-for-testing',
+            },
+        });
+    }
+
+    if (datasetId === '404') {
+        return res.status(404).json({ error: { type: 'record-not-found' } });
+    }
+
+    return res.json({
+        data: {
+            id: 'get-dataset',
+        },
+    });
+});
+
 module.exports = datasets;

test/mock_server/routes/datasets.js

@@ -7,7 +7,6 @@ const keyValueStores = express.Router();
 const ROUTES = [
     { id: 'list-stores', method: 'GET', path: '/' },
     { id: 'get-or-create-store', method: 'POST', path: '/' },
-    { id: 'get-store', method: 'GET', path: '/:storeId' },
     { id: 'delete-store', method: 'DELETE', path: '/:storeId' },
     { id: 'update-store', method: 'PUT', path: '/:storeId' },
     { id: 'get-record', method: 'GET', path: '/:storeId/records/:key', type: 'responseJsonMock' },
@@ -24,4 +23,34 @@ const ROUTES = [
 
 addRoutes(keyValueStores, ROUTES);
 
+/**
+ * GET /key-value-stores/:storeId
+ * Returns a specific key-value store by its ID.
+ * If the store ID is 'id-with-secret-key', it returns a store with a URL signing secret key.
+ * If the store ID is '404', it returns a 404 error with a RECORD_NOT_FOUND type.
+ * Otherwise, it returns a store with an ID of 'get-store' (default).
+ */
+keyValueStores.get('/:storeId', (req, res) => {
+    const { storeId } = req.params;
+
+    if (storeId === 'id-with-secret-key') {
+        return res.json({
+            data: {
+                id: storeId,
+                urlSigningSecretKey: 'secret-key-for-testing',
+            },
+        });
+    }
+
+    if (storeId === '404') {
+        return res.status(404).json({ error: { type: 'record-not-found' } });
+    }
+
+    return res.json({
+        data: {
+            id: 'get-store',
+        },
+    });
+});
+
 module.exports = keyValueStores;