Skip to content

Commit a1e9982

Browse files
danpoletaevvladfrangu
danpoletaev
and
vladfrangu
authored
feat: add signature to KV store URLs where required (#875)
Co-authored-by: Vlad Frangu <[email protected]>
1 parent 27ccb89 commit a1e9982

File tree

5 files changed

+93
-69
lines changed

5 files changed

+93
-69
lines changed

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@
6666
"@apify/actor-templates": "^0.1.5",
6767
"@apify/consts": "^2.36.0",
6868
"@apify/input_schema": "^3.17.0",
69-
"@apify/utilities": "^2.15.1",
69+
"@apify/utilities": "^2.18.0",
7070
"@crawlee/memory-storage": "^3.12.0",
7171
"@inquirer/core": "^10.1.15",
7272
"@inquirer/input": "^4.2.1",
@@ -79,7 +79,7 @@
7979
"@skyra/jaro-winkler": "^1.1.1",
8080
"adm-zip": "~0.5.15",
8181
"ajv": "~8.17.1",
82-
"apify-client": "^2.12.6",
82+
"apify-client": "^2.14.0",
8383
"archiver": "~7.0.1",
8484
"axios": "^1.11.0",
8585
"chalk": "~5.5.0",

‎src/commands/actor/get-public-url.ts‎

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1+
import type { ApifyClient } from 'apify-client';
2+
13
import { ACTOR_ENV_VARS, APIFY_ENV_VARS } from '@apify/consts';
2-
import { createHmacSignature } from '@apify/utilities';
34

45
import { getApifyStorageClient } from '../../lib/actor.js';
56
import { ApifyCommand } from '../../lib/command-framework/apify-command.js';
@@ -27,8 +28,6 @@ export class ActorGetPublicUrlCommand extends ApifyCommand<typeof ActorGetPublic
2728
process.exitCode = CommandExitCodes.NotImplemented;
2829
return;
2930
}
30-
31-
const apiBase = process.env[APIFY_ENV_VARS.API_PUBLIC_BASE_URL];
3231
const storeId = process.env[ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];
3332

3433
// This should never happen, but handle it gracefully to prevent crashes.
@@ -40,11 +39,9 @@ export class ActorGetPublicUrlCommand extends ApifyCommand<typeof ActorGetPublic
4039
return;
4140
}
4241

43-
const apifyClient = await getApifyStorageClient();
42+
const apifyClient = (await getApifyStorageClient()) as ApifyClient;
4443
const store = await apifyClient.keyValueStore(storeId).get();
4544

46-
const publicUrl = new URL(`${apiBase}/v2/key-value-stores/${storeId}/records/${key}`);
47-
4845
if (!store) {
4946
error({
5047
message: `Key-Value store with ID '${storeId}' was not found. Ensure the store exists and that the correct ID is set in ${ACTOR_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID}.`,
@@ -53,13 +50,8 @@ export class ActorGetPublicUrlCommand extends ApifyCommand<typeof ActorGetPublic
5350
return;
5451
}
5552

56-
// @ts-expect-error Add types to client
57-
const { urlSigningSecretKey } = store;
58-
59-
if (urlSigningSecretKey) {
60-
publicUrl.searchParams.append('signature', createHmacSignature(urlSigningSecretKey as string, key));
61-
}
53+
const publicTarballUrl = await apifyClient.keyValueStore(storeId).getRecordPublicUrl(key);
6254

63-
console.log(publicUrl.toString());
55+
console.log(publicTarballUrl);
6456
}
6557
}

‎src/commands/actors/push.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import open from 'open';
77

88
import { fetchManifest } from '@apify/actor-templates';
99
import { ACTOR_JOB_STATUSES, ACTOR_SOURCE_TYPES, MAX_MULTIFILE_BYTES } from '@apify/consts';
10+
import { createHmacSignature } from '@apify/utilities';
1011

1112
import { ApifyCommand } from '../../lib/command-framework/apify-command.js';
1213
import { Args } from '../../lib/command-framework/args.js';
@@ -255,7 +256,24 @@ Skipping push. Use --force to override.`,
255256
contentType: 'application/zip',
256257
});
257258
unlinkSync(TEMP_ZIP_FILE_NAME);
258-
tarballUrl = `${apifyClient.baseUrl}/key-value-stores/${store.id}/records/${key}?disableRedirect=true`;
259+
const tempTarballUrl = new URL(
260+
`${apifyClient.baseUrl}/key-value-stores/${store.id}/records/${key}?disableRedirect=true`,
261+
);
262+
263+
/**
264+
* Signs the tarball URL to grant temporary access for restricted resources.
265+
* When a store is set to 'RESTRICTED', direct URLs are disabled. Instead of
266+
* appending a security token, we add a signature to the URL parameters.
267+
* https://github.com/apify/apify-core/issues/22197
268+
*
269+
* TODO: Use keyValueStore(:storeId).getRecordPublicUrl from apify-client instead once it is released.
270+
*/
271+
if (store?.urlSigningSecretKey) {
272+
const signature = createHmacSignature(store.urlSigningSecretKey, key);
273+
tempTarballUrl.searchParams.set('signature', signature);
274+
}
275+
276+
tarballUrl = tempTarballUrl.toString();
259277
sourceType = ACTOR_SOURCE_TYPES.TARBALL;
260278
}
261279

‎test/api/commands/push.test.ts‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@ import { mkdir, writeFile } from 'node:fs/promises';
33

44
import type { ActorCollectionCreateOptions } from 'apify-client';
55

6-
import { ACTOR_SOURCE_TYPES, SOURCE_FILE_FORMATS } from '@apify/consts';
6+
import { ACTOR_SOURCE_TYPES, SOURCE_FILE_FORMATS, STORAGE_GENERAL_ACCESS } from '@apify/consts';
7+
import { createHmacSignature } from '@apify/utilities';
78

89
import { testRunCommand } from '../../../src/lib/command-framework/apify-command.js';
910
import { LOCAL_CONFIG_PATH } from '../../../src/lib/consts.js';
@@ -236,7 +237,20 @@ describe('[api] apify push', () => {
236237
testActor = (await testActorClient.get())!;
237238
const testActorVersion = await testActorClient.version(actorJson.version).get();
238239
const store = await testUserClient.keyValueStores().getOrCreate(`actor-${testActor.id}-source`);
239-
await testUserClient.keyValueStore(store.id).delete(); // We just needed the store ID, we can clean up now
240+
241+
// Update the store's general access to RESTRICTED
242+
await testUserClient.keyValueStore(store.id).update({ generalAccess: STORAGE_GENERAL_ACCESS.RESTRICTED });
243+
244+
expect(store.urlSigningSecretKey).toBeDefined();
245+
const signature = createHmacSignature(store.urlSigningSecretKey!, `version-${actorJson.version}.zip`);
246+
247+
// Check if the tarball URL is accessible
248+
await expect(fetch((testActorVersion as { tarballUrl: string }).tarballUrl)).resolves.toHaveProperty(
249+
'status',
250+
200,
251+
);
252+
253+
await testUserClient.keyValueStore(store.id).delete();
240254

241255
if (testActor) await testActorClient.delete();
242256

@@ -245,7 +259,7 @@ describe('[api] apify push', () => {
245259
buildTag: 'latest',
246260
tarballUrl:
247261
`${testActorClient.baseUrl}/key-value-stores/${store.id}` +
248-
`/records/version-${actorJson.version}.zip?disableRedirect=true`,
262+
`/records/version-${actorJson.version}.zip?disableRedirect=true&signature=${signature}`,
249263
envVars: testActorWithEnvVars.versions[0].envVars,
250264
sourceType: ACTOR_SOURCE_TYPES.TARBALL,
251265
});

‎yarn.lock‎

Lines changed: 50 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@ __metadata:
1212
languageName: node
1313
linkType: hard
1414

15-
"@apify/consts@npm:^2.20.0, @apify/consts@npm:^2.23.0, @apify/consts@npm:^2.25.0, @apify/consts@npm:^2.36.0, @apify/consts@npm:^2.44.0":
16-
version: 2.44.0
17-
resolution: "@apify/consts@npm:2.44.0"
18-
checksum: 10c0/3b6c1baf7c02b59b5ab04a73c66732c392ed5e96afcd79dd71c42b1c97a61857bc5656d9d312cced0ec4541c43a526148abafe1095b316a1c004bd5f95722999
15+
"@apify/consts@npm:^2.20.0, @apify/consts@npm:^2.23.0, @apify/consts@npm:^2.25.0, @apify/consts@npm:^2.36.0, @apify/consts@npm:^2.44.1":
16+
version: 2.44.1
17+
resolution: "@apify/consts@npm:2.44.1"
18+
checksum: 10c0/3d6c5df96d9338f51acd3bf0063f92f21030d4a79e40653d949c12f9c4077366d087505ffb5859199c492f31f93538566961471a96610b756fbbb0a5519b6ad7
1919
languageName: node
2020
linkType: hard
2121

@@ -49,37 +49,37 @@ __metadata:
4949
linkType: hard
5050

5151
"@apify/input_schema@npm:^3.17.0":
52-
version: 3.18.3
53-
resolution: "@apify/input_schema@npm:3.18.3"
52+
version: 3.18.4
53+
resolution: "@apify/input_schema@npm:3.18.4"
5454
dependencies:
55-
"@apify/consts": "npm:^2.44.0"
56-
"@apify/input_secrets": "npm:^1.2.4"
55+
"@apify/consts": "npm:^2.44.1"
56+
"@apify/input_secrets": "npm:^1.2.5"
5757
acorn-loose: "npm:^8.4.0"
5858
countries-list: "npm:^3.0.0"
5959
peerDependencies:
6060
ajv: ^8.0.0
61-
checksum: 10c0/6c8a7acd60007baeece5e7149585e193ecdab49c98436ad54e578be4e7adbb0db428d9dda28b492af570f4ed8e70690ccb8bca1fe460e24443cfe4a812940b1d
61+
checksum: 10c0/6d2b0e3aa38a553a26db2fc389354e5ec130243eabcb21050a65183fd1ba3b003ec271d6fd5806287ff84adb79ce3559ec188fea245f7c3bbda50d80ae449b88
6262
languageName: node
6363
linkType: hard
6464

65-
"@apify/input_secrets@npm:^1.2.0, @apify/input_secrets@npm:^1.2.4":
66-
version: 1.2.4
67-
resolution: "@apify/input_secrets@npm:1.2.4"
65+
"@apify/input_secrets@npm:^1.2.0, @apify/input_secrets@npm:^1.2.5":
66+
version: 1.2.5
67+
resolution: "@apify/input_secrets@npm:1.2.5"
6868
dependencies:
69-
"@apify/log": "npm:^2.5.21"
70-
"@apify/utilities": "npm:^2.18.1"
69+
"@apify/log": "npm:^2.5.22"
70+
"@apify/utilities": "npm:^2.18.2"
7171
ow: "npm:^0.28.2"
72-
checksum: 10c0/6a71ea85c6b9cabf2b29f0ab2e3acd42bb9b9eac8371699dc7ed3d706ae4204de97100fd764800fa038ea2439a8ff422e3dff214edf9c8db2cf8d8cf2ec29ffe
72+
checksum: 10c0/6ce482850140dc25cd6a5c6f0c76c33c168686b8f6470aa8efb7557f2c5b42ab8cb2ab9d0e503bbc9e9d9c3df6325cf66886c33cb2b776e5061e545c85df0b4e
7373
languageName: node
7474
linkType: hard
7575

76-
"@apify/log@npm:^2.2.6, @apify/log@npm:^2.4.0, @apify/log@npm:^2.4.3, @apify/log@npm:^2.5.21":
77-
version: 2.5.21
78-
resolution: "@apify/log@npm:2.5.21"
76+
"@apify/log@npm:^2.2.6, @apify/log@npm:^2.4.0, @apify/log@npm:^2.4.3, @apify/log@npm:^2.5.22":
77+
version: 2.5.22
78+
resolution: "@apify/log@npm:2.5.22"
7979
dependencies:
80-
"@apify/consts": "npm:^2.44.0"
80+
"@apify/consts": "npm:^2.44.1"
8181
ansi-colors: "npm:^4.1.1"
82-
checksum: 10c0/6115c9bfc36bcc2ba5812cec66fd8c8a5e62e7d4177460e41cd68d92da1704def067359c715f39de5c4238f220ba6ec7caeb402467106b7d116789d872fbdb71
82+
checksum: 10c0/10ba3b00b1b57fb49adfd6800e5d2fc46c471eb442510aedf47edb303fedde9164e527022e0f3bd5f78d1778025084d13145e45d15aa1955afe086d6b2bbcf84
8383
languageName: node
8484
linkType: hard
8585

@@ -95,11 +95,11 @@ __metadata:
9595
linkType: hard
9696

9797
"@apify/pseudo_url@npm:^2.0.30":
98-
version: 2.0.62
99-
resolution: "@apify/pseudo_url@npm:2.0.62"
98+
version: 2.0.63
99+
resolution: "@apify/pseudo_url@npm:2.0.63"
100100
dependencies:
101-
"@apify/log": "npm:^2.5.21"
102-
checksum: 10c0/d0919998055781f4304794db4973fd00d4a7f47fa8fc94ad39c087d7e0901a547722f0b7f9081dd5da0d57af7129514c09b9f917bab62dd55a9f46df2d7818c2
101+
"@apify/log": "npm:^2.5.22"
102+
checksum: 10c0/fe4170e099db2af5ce6e6ebc3fba31d1838c5bceb6bd239eb93b50d87a5e2fbf36959200518a030e97ff2627d7c4784f5ae9216fe56e6519fdf0f0c2c1df7edf
103103
languageName: node
104104
linkType: hard
105105

@@ -117,13 +117,13 @@ __metadata:
117117
languageName: node
118118
linkType: hard
119119

120-
"@apify/utilities@npm:^2.13.0, @apify/utilities@npm:^2.15.1, @apify/utilities@npm:^2.18.0, @apify/utilities@npm:^2.18.1, @apify/utilities@npm:^2.7.10":
121-
version: 2.18.1
122-
resolution: "@apify/utilities@npm:2.18.1"
120+
"@apify/utilities@npm:^2.13.0, @apify/utilities@npm:^2.18.0, @apify/utilities@npm:^2.18.2, @apify/utilities@npm:^2.7.10":
121+
version: 2.18.2
122+
resolution: "@apify/utilities@npm:2.18.2"
123123
dependencies:
124-
"@apify/consts": "npm:^2.44.0"
125-
"@apify/log": "npm:^2.5.21"
126-
checksum: 10c0/082f4ab78d38478e9a3c7c3bc876196b3bbfae863c25f0c643e933ddbc9637c9f58155d5208f328d36011d2e7507393e33f77c3784a760b691a9d8230877f246
124+
"@apify/consts": "npm:^2.44.1"
125+
"@apify/log": "npm:^2.5.22"
126+
checksum: 10c0/1f3d24ca9470dbf3bfb3539daf7e58cbd36debf18b4d9e4ff640b4b692b7435492a84c735bc082bbc6d28a96750251e9846c49ea4553cc0f90c610605411ebeb
127127
languageName: node
128128
linkType: hard
129129

@@ -500,14 +500,14 @@ __metadata:
500500
linkType: hard
501501

502502
"@cucumber/query@npm:^13.0.2":
503-
version: 13.5.0
504-
resolution: "@cucumber/query@npm:13.5.0"
503+
version: 13.6.0
504+
resolution: "@cucumber/query@npm:13.6.0"
505505
dependencies:
506506
"@teppeis/multimaps": "npm:3.0.0"
507507
lodash.sortby: "npm:^4.7.0"
508508
peerDependencies:
509509
"@cucumber/messages": "*"
510-
checksum: 10c0/af8ba3491fd5de38aab964578e7b744c9c02f03c0f2a2101c80da4d628160b6c1e6fd8f1edcec30c3200bb9b1d1074a38e8b420b58fda873a7e66e9b032f3479
510+
checksum: 10c0/a8df203c590bdd2da3c86b10abbe7db0da0391e1920eeb742fad944c8b42e5af1d2f93f99443b8ae7949698b29c340558e9e2709d88bfc71767bafe049fc3141
511511
languageName: node
512512
linkType: hard
513513

@@ -1374,11 +1374,11 @@ __metadata:
13741374
linkType: hard
13751375

13761376
"@types/bun@npm:^1.2.5":
1377-
version: 1.2.19
1378-
resolution: "@types/bun@npm:1.2.19"
1377+
version: 1.2.20
1378+
resolution: "@types/bun@npm:1.2.20"
13791379
dependencies:
1380-
bun-types: "npm:1.2.19"
1381-
checksum: 10c0/c5bdc7c25fd7d33fe9b27f0065c9f0386e8ca61bb141e3d3f1b799c76a71ac2c3a0984b10999d092af5809ea67434f7b5fab97942d5630a07cbff0ae9c58b49a
1380+
bun-types: "npm:1.2.20"
1381+
checksum: 10c0/62972bde7496b804ae166cbe859579b4770a8e9d44bb87d5ae38e232c7b42a98e1acc93a8d6533449385a7c8daf437d71a7d0cd80b42e5035ff21563befbb2e7
13821382
languageName: node
13831383
linkType: hard
13841384

@@ -2237,7 +2237,7 @@ __metadata:
22372237
"@apify/eslint-config": "npm:^1.0.0"
22382238
"@apify/input_schema": "npm:^3.17.0"
22392239
"@apify/tsconfig": "npm:^0.1.1"
2240-
"@apify/utilities": "npm:^2.15.1"
2240+
"@apify/utilities": "npm:^2.18.0"
22412241
"@biomejs/biome": "npm:^2.0.0"
22422242
"@crawlee/memory-storage": "npm:^3.12.0"
22432243
"@crawlee/types": "npm:^3.11.1"
@@ -2271,7 +2271,7 @@ __metadata:
22712271
adm-zip: "npm:~0.5.15"
22722272
ajv: "npm:~8.17.1"
22732273
apify: "npm:^3.2.4"
2274-
apify-client: "npm:^2.12.6"
2274+
apify-client: "npm:^2.14.0"
22752275
archiver: "npm:~7.0.1"
22762276
axios: "npm:^1.11.0"
22772277
chai: "npm:^4.4.1"
@@ -2321,9 +2321,9 @@ __metadata:
23212321
languageName: unknown
23222322
linkType: soft
23232323

2324-
"apify-client@npm:^2.12.1, apify-client@npm:^2.12.6":
2325-
version: 2.13.0
2326-
resolution: "apify-client@npm:2.13.0"
2324+
"apify-client@npm:^2.12.1, apify-client@npm:^2.14.0":
2325+
version: 2.14.0
2326+
resolution: "apify-client@npm:2.14.0"
23272327
dependencies:
23282328
"@apify/consts": "npm:^2.25.0"
23292329
"@apify/log": "npm:^2.2.6"
@@ -2336,7 +2336,7 @@ __metadata:
23362336
ow: "npm:^0.28.2"
23372337
tslib: "npm:^2.5.0"
23382338
type-fest: "npm:^4.0.0"
2339-
checksum: 10c0/0596ba6aa4a534fc514e4d32b5ed232f1be7fd49cd3759380332edd136f5697add34a9e2ff3f8e98088f9945faf64ea5a7ebe53eb3d7b3886333ab3eddddfa32
2339+
checksum: 10c0/05eec1e28baa5ff8c8b4273e2671c8075aee1f53b40dad2d00d9d6ac96fdda2e1fe9a62708efa7f8bfc89c09719108822c71dd20ff0e44d5339c35a1ece37578
23402340
languageName: node
23412341
linkType: hard
23422342

@@ -2690,14 +2690,14 @@ __metadata:
26902690
languageName: node
26912691
linkType: hard
26922692

2693-
"bun-types@npm:1.2.19":
2694-
version: 1.2.19
2695-
resolution: "bun-types@npm:1.2.19"
2693+
"bun-types@npm:1.2.20":
2694+
version: 1.2.20
2695+
resolution: "bun-types@npm:1.2.20"
26962696
dependencies:
26972697
"@types/node": "npm:*"
26982698
peerDependencies:
26992699
"@types/react": ^19
2700-
checksum: 10c0/5475ad5c3f224244115c98da1329bc2ce700884d589eac487eaa111599361ed742c1041eefc0af1a8065dc9f722a6cbfaafe70aa049fcba1c3136f8ec61c4f52
2700+
checksum: 10c0/cea622d818d296df77a9a4c88b9b01e35953b0c82a816d3d64b139aa42b3948841b2cd696a08422ea44cc5b71b92385712736083d973159c938600cb369b8ac4
27012701
languageName: node
27022702
linkType: hard
27032703

@@ -2853,9 +2853,9 @@ __metadata:
28532853
linkType: hard
28542854

28552855
"caniuse-lite@npm:^1.0.30001733":
2856-
version: 1.0.30001733
2857-
resolution: "caniuse-lite@npm:1.0.30001733"
2858-
checksum: 10c0/2c03ad3362be7c93c09537f3853156ade5c70fb131888971dd538631971873ba27b83c5aad48dd06b6cde005fe57caae2db5d0f467d0c63a315391add70f01f5
2856+
version: 1.0.30001734
2857+
resolution: "caniuse-lite@npm:1.0.30001734"
2858+
checksum: 10c0/5869cb6a01e7a012a8c5d7b0482e2c910be3a2a469d4ef516a54db3f846fbaedb2600eeaa270dae9e2ad9328e33f39782e6f459405fcca620021f5f06694542d
28592859
languageName: node
28602860
linkType: hard
28612861

0 commit comments

Comments
 (0)