Skip to content

Commit 6849201

Browse files
PierreJeanjacquotPierreJeanjacquot
committed
refactor: split processBulkRequest in 2 methods
1 parent 9898bcb commit 6849201

File tree

9 files changed

+461
-318
lines changed

9 files changed

+461
-318
lines changed

‎packages/sdk/package-lock.json‎

Lines changed: 17 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/sdk/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
"debug": "^4.3.4",
6262
"ethers": "^6.13.2",
6363
"graphql-request": "^6.0.0",
64-
"iexec": "^8.20.0-feat-bulk-processing-d3655be",
64+
"iexec": "^8.20.0-feat-bulk-processing-62e7c5f",
6565
"jszip": "^3.7.1",
6666
"kubo-rpc-client": "^4.1.1",
6767
"magic-bytes.js": "^1.0.15",

‎packages/sdk/src/lib/dataProtectorCore/IExecDataProtectorCore.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,16 @@ import {
2121
TransferResponse,
2222
WaitForTaskCompletionResponse,
2323
WaitForTaskCompletionParams,
24+
PrepareBulkRequestParams,
25+
PrepareBulkRequestResponse,
2426
} from '../types/index.js';
2527
import { getGrantedAccess } from './getGrantedAccess.js';
2628
import { getProtectedData } from './getProtectedData.js';
2729
import { getResultFromCompletedTask } from './getResultFromCompletedTask.js';
2830
import { grantAccess } from './grantAccess.js';
29-
import { processProtectedData } from './processProtectedData.js';
31+
import { prepareBulkRequest } from './prepareBulkRequest.js';
3032
import { processBulkRequest } from './processBulkRequest.js';
33+
import { processProtectedData } from './processProtectedData.js';
3134
import { protectData } from './protectData.js';
3235
import { revokeAllAccess } from './revokeAllAccess.js';
3336
import { revokeOneAccess } from './revokeOneAccess.js';
@@ -86,6 +89,18 @@ class IExecDataProtectorCore extends IExecDataProtectorModule {
8689
});
8790
}
8891

92+
async prepareBulkRequest(
93+
args: PrepareBulkRequestParams
94+
): Promise<PrepareBulkRequestResponse> {
95+
await this.init();
96+
await isValidProvider(this.iexec);
97+
return prepareBulkRequest({
98+
...args,
99+
iexec: this.iexec,
100+
defaultWorkerpool: this.defaultWorkerpool,
101+
});
102+
}
103+
89104
async processBulkRequest(
90105
args: ProcessBulkRequestParams
91106
): Promise<ProcessBulkRequestResponse> {

‎packages/sdk/src/lib/dataProtectorCore/prepareBulkRequest.ts‎

Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
import { utils } from 'iexec';
2+
import { NULL_ADDRESS } from 'iexec/utils';
3+
import {
4+
MAX_DESIRED_APP_ORDER_PRICE,
5+
MAX_DESIRED_WORKERPOOL_ORDER_PRICE,
6+
SCONE_TAG,
7+
} from '../../config/config.js';
8+
import {
9+
WorkflowError,
10+
handleIfProtocolError,
11+
prepareBulkRequestErrorMessage,
12+
} from '../../utils/errors.js';
13+
import { pushRequesterSecret } from '../../utils/pushRequesterSecret.js';
14+
import {
15+
getPemFormattedKeyPair,
16+
formatPemPublicKeyForSMS,
17+
} from '../../utils/rsa.js';
18+
import {
19+
addressOrEnsSchema,
20+
booleanSchema,
21+
positiveNumberSchema,
22+
secretsSchema,
23+
stringSchema,
24+
throwIfMissing,
25+
urlArraySchema,
26+
validateOnStatusUpdateCallback,
27+
} from '../../utils/validators.js';
28+
import {
29+
DefaultWorkerpoolConsumer,
30+
OnStatusUpdateFn,
31+
PrepareBulkRequestParams,
32+
PrepareBulkRequestResponse,
33+
PrepareBulkRequestStatuses,
34+
} from '../types/index.js';
35+
import { IExecConsumer } from '../types/internalTypes.js';
36+
37+
export type PrepareBulkRequest = typeof prepareBulkRequest;
38+
39+
export const prepareBulkRequest = async ({
40+
iexec = throwIfMissing(),
41+
bulkOrders,
42+
app,
43+
maxProtectedDataPerTask,
44+
appMaxPrice = MAX_DESIRED_APP_ORDER_PRICE,
45+
workerpoolMaxPrice = MAX_DESIRED_WORKERPOOL_ORDER_PRICE,
46+
args,
47+
inputFiles,
48+
secrets,
49+
workerpool,
50+
encryptResult = false,
51+
pemPrivateKey,
52+
onStatusUpdate = () => {},
53+
}: IExecConsumer &
54+
DefaultWorkerpoolConsumer &
55+
PrepareBulkRequestParams): Promise<PrepareBulkRequestResponse> => {
56+
const vBulkOrders = bulkOrders;
57+
if (!vBulkOrders || vBulkOrders.length === 0) {
58+
throw new Error('bulkOrders is required and must not be empty');
59+
}
60+
const vApp = addressOrEnsSchema().required().label('app').validateSync(app);
61+
const vMaxProtectedDataPerTask = positiveNumberSchema()
62+
.required()
63+
.label('maxProtectedDataPerTask')
64+
.validateSync(maxProtectedDataPerTask);
65+
const vAppMaxPrice = positiveNumberSchema()
66+
.label('appMaxPrice')
67+
.validateSync(appMaxPrice);
68+
const vWorkerpoolMaxPrice = positiveNumberSchema()
69+
.label('workerpoolMaxPrice')
70+
.validateSync(workerpoolMaxPrice);
71+
const vInputFiles = urlArraySchema()
72+
.label('inputFiles')
73+
.validateSync(inputFiles);
74+
const vArgs = stringSchema().label('args').validateSync(args);
75+
const vSecrets = secretsSchema().label('secrets').validateSync(secrets);
76+
const vWorkerpool = addressOrEnsSchema()
77+
.default(NULL_ADDRESS)
78+
.label('workerpool')
79+
.validateSync(workerpool);
80+
const vEncryptResult = booleanSchema()
81+
.label('encryptResult')
82+
.validateSync(encryptResult);
83+
const vPemPrivateKey = stringSchema()
84+
.label('pemPrivateKey')
85+
.validateSync(pemPrivateKey);
86+
const vOnStatusUpdate =
87+
validateOnStatusUpdateCallback<
88+
OnStatusUpdateFn<PrepareBulkRequestStatuses>
89+
>(onStatusUpdate);
90+
91+
// Validate that if pemPrivateKey is provided, encryptResult must be true
92+
if (vPemPrivateKey && !vEncryptResult) {
93+
throw new Error(
94+
'pemPrivateKey can only be provided when encryptResult is true'
95+
);
96+
}
97+
98+
// TODO validate bulkOrders?
99+
// price, volume, app, workerpool, requester, signature (including whitelists)
100+
101+
try {
102+
vOnStatusUpdate({
103+
title: 'PUSH_REQUESTER_SECRET',
104+
isDone: false,
105+
});
106+
const secretsId = await pushRequesterSecret(iexec, vSecrets);
107+
vOnStatusUpdate({
108+
title: 'PUSH_REQUESTER_SECRET',
109+
isDone: true,
110+
});
111+
112+
vOnStatusUpdate({
113+
title: 'GENERATE_ENCRYPTION_KEY',
114+
isDone: false,
115+
});
116+
117+
// Handle result encryption
118+
let privateKey: string | undefined;
119+
if (vEncryptResult) {
120+
if (!vPemPrivateKey) {
121+
vOnStatusUpdate({
122+
title: 'GENERATE_ENCRYPTION_KEY',
123+
isDone: false,
124+
});
125+
}
126+
const pemKeyPair = await getPemFormattedKeyPair({
127+
pemPrivateKey: vPemPrivateKey,
128+
});
129+
privateKey = pemKeyPair.pemPrivateKey;
130+
// Notify user if a new key was generated
131+
if (!vPemPrivateKey) {
132+
vOnStatusUpdate({
133+
title: 'GENERATE_ENCRYPTION_KEY',
134+
isDone: true,
135+
payload: {
136+
pemPrivateKey: pemKeyPair.pemPrivateKey,
137+
},
138+
});
139+
}
140+
141+
vOnStatusUpdate({
142+
title: 'PUSH_ENCRYPTION_KEY',
143+
isDone: false,
144+
payload: {
145+
pemPublicKey: pemKeyPair.pemPublicKey,
146+
},
147+
});
148+
149+
await iexec.result.pushResultEncryptionKey(
150+
formatPemPublicKeyForSMS(pemKeyPair.pemPublicKey),
151+
{
152+
forceUpdate: true,
153+
}
154+
);
155+
156+
vOnStatusUpdate({
157+
title: 'PUSH_ENCRYPTION_KEY',
158+
isDone: true,
159+
payload: {
160+
pemPublicKey: pemKeyPair.pemPublicKey,
161+
},
162+
});
163+
}
164+
165+
// Prepare dataset bulk
166+
vOnStatusUpdate({
167+
title: 'PREPARE_PROTECTED_DATA_BULK',
168+
isDone: false,
169+
});
170+
171+
const { cid, volume } = await iexec.order.prepareDatasetBulk(vBulkOrders, {
172+
maxDatasetPerTask: parseInt(vMaxProtectedDataPerTask.toString()),
173+
});
174+
175+
vOnStatusUpdate({
176+
title: 'PREPARE_PROTECTED_DATA_BULK',
177+
isDone: true,
178+
});
179+
180+
// Create request order for the whole bulk (only once)
181+
vOnStatusUpdate({
182+
title: 'CREATE_REQUEST_ORDER',
183+
isDone: false,
184+
});
185+
186+
const requestorderToSign = await iexec.order.createRequestorder({
187+
app: vApp,
188+
appmaxprice: vAppMaxPrice,
189+
workerpool: vWorkerpool,
190+
workerpoolmaxprice: vWorkerpoolMaxPrice,
191+
volume,
192+
category: 0,
193+
tag: SCONE_TAG,
194+
params: {
195+
bulk_cid: cid,
196+
iexec_input_files: vInputFiles,
197+
iexec_secrets: secretsId,
198+
iexec_args: vArgs,
199+
...(vEncryptResult ? { iexec_result_encryption: true } : {}),
200+
},
201+
trust: 0,
202+
// bulk order
203+
datasetmaxprice: 0,
204+
dataset: utils.NULL_ADDRESS,
205+
});
206+
207+
const requestorder = await iexec.order.signRequestorder(requestorderToSign);
208+
209+
vOnStatusUpdate({
210+
title: 'CREATE_REQUEST_ORDER',
211+
isDone: true,
212+
});
213+
214+
return {
215+
bulkRequest: requestorder,
216+
...(privateKey ? { pemPrivateKey: privateKey } : {}),
217+
};
218+
} catch (error) {
219+
console.error('[prepareBulkRequest] ERROR', error);
220+
handleIfProtocolError(error);
221+
throw new WorkflowError({
222+
message: prepareBulkRequestErrorMessage,
223+
errorCause: error,
224+
});
225+
}
226+
};

0 commit comments

Comments
 (0)