feat: allowed signature to be passed in kv-store/datasets (#761)

When working with output schema in UI I found that we do not allow
passing signature in client to read data from storages. This PR does not
fully resolve it, but it resolves most of the issues I found.

I will leave the rest to Daniil, since he might now better which
endpoints use signatures and which do not.

Author: gippy

src/resource_clients/dataset.ts

@@ -69,6 +69,7 @@ export class DatasetClient<
                 skipHidden: ow.optional.boolean,
                 unwind: ow.optional.any(ow.string, ow.array.ofType(ow.string)),
                 view: ow.optional.string,
+                signature: ow.optional.string,
             }),
         );
 
@@ -109,6 +110,7 @@ export class DatasetClient<
                 view: ow.optional.string,
                 xmlRoot: ow.optional.string,
                 xmlRow: ow.optional.string,
+                signature: ow.optional.string,
             }),
         );
 
@@ -272,6 +274,7 @@ export interface DatasetClientListItemOptions {
     skipHidden?: boolean;
     unwind?: string | string[]; // TODO: when doing a breaking change release, change to string[] only
     view?: string;
+    signature?: string;
 }
 
 export interface DatasetClientCreateItemsUrlOptions extends DatasetClientListItemOptions {

src/resource_clients/key_value_store.ts

@@ -72,6 +72,7 @@ export class KeyValueStoreClient extends ResourceClient {
                 exclusiveStartKey: ow.optional.string,
                 collection: ow.optional.string,
                 prefix: ow.optional.string,
+                signature: ow.optional.string,
             }),
         );
 
@@ -201,6 +202,7 @@ export class KeyValueStoreClient extends ResourceClient {
                 buffer: ow.optional.boolean,
                 stream: ow.optional.boolean,
                 disableRedirect: ow.optional.boolean,
+                signature: ow.optional.string,
             }),
         );
 
@@ -215,10 +217,13 @@ export class KeyValueStoreClient extends ResourceClient {
             );
         }
 
+        const queryParams: Pick<KeyValueClientGetRecordOptions, 'signature'> = {};
+        if (options.signature) queryParams.signature = options.signature;
+
         const requestOpts: Record<string, unknown> = {
             url: this._url(`records/${key}`),
             method: 'GET',
-            params: this._params(),
+            params: this._params(queryParams),
             timeout: DEFAULT_TIMEOUT_MILLIS,
         };
 
@@ -353,6 +358,7 @@ export interface KeyValueClientListKeysOptions {
     exclusiveStartKey?: string;
     collection?: string;
     prefix?: string;
+    signature?: string;
 }
 
 export interface KeyValueClientCreateKeysUrlOptions extends KeyValueClientListKeysOptions {
@@ -377,6 +383,7 @@ export interface KeyValueListItem {
 export interface KeyValueClientGetRecordOptions {
     buffer?: boolean;
     stream?: boolean;
+    signature?: string;
 }
 
 export interface KeyValueStoreRecord<T> {

test/datasets.test.js

@@ -224,6 +224,28 @@ describe('Dataset methods', () => {
             validateRequest(qs, { datasetId });
         });
 
+        test('listItems() correctly passes signature', async () => {
+            const datasetId = 'some-id';
+            const body = [{ test: 'value' }];
+            const headers = {
+                'content-type': 'application/json; chartset=utf-8',
+                'x-apify-pagination-total': '1',
+                'x-apify-pagination-offset': '1',
+                'x-apify-pagination-count': '0',
+                'x-apify-pagination-limit': '1',
+                // TODO: https://github.com/apify/apify-core/issues/3503
+                'x-apify-pagination-desc': false,
+            };
+            const qs = { limit: 1, offset: 1, signature: 'some-signature' };
+            mockServer.setResponse({ body, headers });
+
+            await client.dataset(datasetId).listItems(qs);
+            validateRequest(qs, { datasetId });
+
+            await page.evaluate((id, opts) => client.dataset(id).listItems(opts), datasetId, qs);
+            validateRequest(qs, { datasetId });
+        });
+
         test("downloadItems() doesn't parse application/json", async () => {
             const datasetId = 'some-id';
             const body = JSON.stringify({ a: 'foo', b: ['bar1', 'bar2'] });
@@ -258,6 +280,36 @@ describe('Dataset methods', () => {
             validateRequest({ format, ...options }, { datasetId });
         });
 
+        test('downloadItems() correctly passes signature', async () => {
+            const datasetId = 'some-id';
+            const body = JSON.stringify({ a: 'foo', b: ['bar1', 'bar2'] });
+            const format = 'json';
+            const options = {
+                bom: true,
+                signature: 'some-signature',
+            };
+            const headers = {
+                contentType: 'application/json; charset=utf-8',
+            };
+
+            mockServer.setResponse({ body, headers });
+
+            await client.dataset(datasetId).downloadItems(format, options);
+            validateRequest({ format, ...options }, { datasetId });
+
+            await page.evaluate(
+                async (id, f, opts) => {
+                    const res = await client.dataset(id).downloadItems(f, opts);
+                    const decoder = new TextDecoder();
+                    return decoder.decode(res);
+                },
+                datasetId,
+                format,
+                options,
+            );
+            validateRequest({ format, ...options }, { datasetId });
+        });
+
         test('pushItems() works with object', async () => {
             const datasetId = '201';
             const data = { someData: 'someValue' };

test/key_value_stores.test.js

@@ -146,6 +146,24 @@ describe('Key-Value Store methods', () => {
             validateRequest(query, { storeId });
         });
 
+        test('listKeys() passes signature', async () => {
+            const storeId = 'some-id';
+
+            const query = {
+                limit: 10,
+                exclusiveStartKey: 'fromKey',
+                collection: 'my-collection',
+                prefix: 'my-prefix',
+                signature: 'some-signature',
+            };
+
+            await client.keyValueStore(storeId).listKeys(query);
+            validateRequest(query, { storeId });
+
+            await page.evaluate((id, opts) => client.keyValueStore(id).listKeys(opts), storeId, query);
+            validateRequest(query, { storeId });
+        });
+
         test('recordExists() works', async () => {
             const key = 'some-key';
             const storeId = 'some-id';
@@ -367,6 +385,24 @@ describe('Key-Value Store methods', () => {
             }
         });
 
+        test('getRecord() correctly passes signature', async () => {
+            const key = 'some-key';
+            const storeId = 'some-id';
+            const options = {
+                signature: 'some-signature',
+            };
+
+            const body = { foo: 'bar', baz: [1, 2] };
+            const expectedContentType = 'application/json; charset=utf-8';
+            const expectedHeaders = {
+                'content-type': expectedContentType,
+            };
+
+            mockServer.setResponse({ headers: expectedHeaders, body });
+            await client.keyValueStore(storeId).getRecord(key, options);
+            validateRequest(options, { storeId, key });
+        });
+
         test('getRecord() returns undefined on 404 status code (RECORD_NOT_FOUND)', async () => {
             const key = 'some-key';
             const storeId = 'some-id';