Feature/update processprotecteddata (#426)

Author: SeddikBellamine

packages/sdk/src/config/config.ts

@@ -23,6 +23,12 @@ export const SCONE_TAG = ['tee', 'scone'];
 
 export const DEFAULT_MAX_PRICE = 0;
 
+export const MAX_DESIRED_DATA_ORDER_PRICE = 0;
+
+export const MAX_DESIRED_APP_ORDER_PRICE = 0;
+
+export const MAX_DESIRED_WORKERPOOL_ORDER_PRICE = 0;
+
 export const KEY_PURPOSE_SELECTOR = 'keyHasPurpose(bytes32,uint256)';
 
 export const GROUP_MEMBER_PURPOSE = 4;

packages/sdk/src/lib/dataProtectorCore/processProtectedData.ts

@@ -1,6 +1,8 @@
 import { ethers } from 'ethers';
 import {
-  DEFAULT_MAX_PRICE,
+  MAX_DESIRED_APP_ORDER_PRICE,
+  MAX_DESIRED_DATA_ORDER_PRICE,
+  MAX_DESIRED_WORKERPOOL_ORDER_PRICE,
   SCONE_TAG,
   WORKERPOOL_ADDRESS,
 } from '../../config/config.js';
@@ -11,7 +13,7 @@ import {
 } from '../../utils/errors.js';
 import {
   checkUserVoucher,
-  findWorkerpoolOrders,
+  filterWorkerpoolOrders,
 } from '../../utils/processProtectedData.models.js';
 import { pushRequesterSecret } from '../../utils/pushRequesterSecret.js';
 import {
@@ -45,7 +47,9 @@ export const processProtectedData = async ({
   protectedData,
   app,
   userWhitelist,
-  maxPrice = DEFAULT_MAX_PRICE,
+  dataMaxPrice = MAX_DESIRED_DATA_ORDER_PRICE,
+  appMaxPrice = MAX_DESIRED_APP_ORDER_PRICE,
+  workerpoolMaxPrice = MAX_DESIRED_WORKERPOOL_ORDER_PRICE,
   path,
   args,
   inputFiles,
@@ -67,9 +71,15 @@ export const processProtectedData = async ({
   const vUserWhitelist = addressSchema()
     .label('userWhitelist')
     .validateSync(userWhitelist);
-  const vMaxPrice = positiveNumberSchema()
-    .label('maxPrice')
-    .validateSync(maxPrice);
+  const vDataMaxPrice = positiveNumberSchema()
+    .label('dataMaxPrice')
+    .validateSync(dataMaxPrice);
+  const vAppMaxPrice = positiveNumberSchema()
+    .label('appMaxPrice')
+    .validateSync(appMaxPrice);
+  const vWorkerpoolMaxPrice = positiveNumberSchema()
+    .label('workerpoolMaxPrice')
+    .validateSync(workerpoolMaxPrice);
   const vPath = stringSchema().label('path').validateSync(path);
   const vInputFiles = urlArraySchema()
     .label('inputFiles')
@@ -136,76 +146,111 @@ export const processProtectedData = async ({
     }
 
     vOnStatusUpdate({
-      title: 'FETCH_PROTECTED_DATA_ORDERBOOK',
+      title: 'FETCH_ORDERS',
       isDone: false,
     });
-    const datasetOrderbook = await iexec.orderbook.fetchDatasetOrderbook(
-      vProtectedData,
-      {
-        app: vApp,
-        workerpool: vWorkerpool,
-        requester,
-      }
-    );
-    const datasetorder = datasetOrderbook.orders[0]?.order; //The first order is the cheapest one
-    if (!datasetorder) {
-      throw new Error(`No dataset orders found`);
-    }
-    vOnStatusUpdate({
-      title: 'FETCH_PROTECTED_DATA_ORDERBOOK',
-      isDone: true,
-    });
+    const [
+      datasetorderForApp,
+      datasetorderForWhitelist,
+      apporder,
+      workerpoolorder,
+    ] = await Promise.all([
+      // Fetch dataset order
+      iexec.orderbook
+        .fetchDatasetOrderbook(vProtectedData, {
+          app: vApp,
+          requester: requester,
+        })
+        .then((datasetOrderbook) => {
+          const desiredPriceDataOrderbook = datasetOrderbook.orders.filter(
+            (order) => order.order.datasetprice <= vDataMaxPrice
+          );
+          return desiredPriceDataOrderbook[0]?.order; // may be undefined
+        }),
+      // Fetch dataset order for whitelist
+      iexec.orderbook
+        .fetchDatasetOrderbook(vProtectedData, {
+          app: vUserWhitelist,
+          requester: requester,
+        })
+        .then((datasetOrderbook) => {
+          const desiredPriceDataOrderbook = datasetOrderbook.orders.filter(
+            (order) => order.order.datasetprice <= vDataMaxPrice
+          );
+          return desiredPriceDataOrderbook[0]?.order; // may be undefined
+        }),
+      // Fetch app order
+      iexec.orderbook
+        .fetchAppOrderbook(vApp, {
+          minTag: ['tee', 'scone'],
+          maxTag: ['tee', 'scone'],
+          workerpool: vWorkerpool,
+        })
+        .then((appOrderbook) => {
+          const desiredPriceAppOrderbook = appOrderbook.orders.filter(
+            (order) => order.order.appprice <= vAppMaxPrice
+          );
+          const desiredPriceAppOrder = desiredPriceAppOrderbook[0]?.order;
+          if (!desiredPriceAppOrder) {
+            throw new Error('No App order found for the desired price');
+          }
+          return desiredPriceAppOrder;
+        }),
+      // Fetch workerpool order for App or AppWhitelist
+      Promise.all([
+        // for app
+        iexec.orderbook.fetchWorkerpoolOrderbook({
+          workerpool: vWorkerpool,
+          app: vApp,
+          dataset: vProtectedData,
+          requester: requester, // public orders + user specific orders
+          isRequesterStrict: useVoucher, // If voucher, we only want user specific orders
+          minTag: ['tee', 'scone'],
+          maxTag: ['tee', 'scone'],
+          category: 0,
+        }),
+        // for app whitelist
+        iexec.orderbook.fetchWorkerpoolOrderbook({
+          workerpool: vWorkerpool === ethers.ZeroAddress ? 'any' : vWorkerpool,
+          app: vUserWhitelist,
+          dataset: vProtectedData,
+          requester: requester, // public orders + user specific orders
+          isRequesterStrict: useVoucher, // If voucher, we only want user specific orders
+          minTag: ['tee', 'scone'],
+          maxTag: ['tee', 'scone'],
+          category: 0,
+        }),
+      ]).then(
+        ([workerpoolOrderbookForApp, workerpoolOrderbookForAppWhitelist]) => {
+          const desiredPriceWorkerpoolOrder = filterWorkerpoolOrders({
+            workerpoolOrders: [
+              ...workerpoolOrderbookForApp.orders,
+              ...workerpoolOrderbookForAppWhitelist.orders,
+            ],
+            workerpoolMaxPrice: vWorkerpoolMaxPrice,
+            useVoucher: vUseVoucher,
+            userVoucher,
+          });
+          if (!desiredPriceWorkerpoolOrder) {
+            throw new Error('No Workerpool order found for the desired price');
+          }
+          return desiredPriceWorkerpoolOrder;
+        }
+      ),
+    ]);
 
-    vOnStatusUpdate({
-      title: 'FETCH_APP_ORDERBOOK',
-      isDone: false,
-    });
-    const appOrderbook = await iexec.orderbook.fetchAppOrderbook(vApp, {
-      dataset: protectedData,
-      requester,
-      minTag: SCONE_TAG,
-      maxTag: SCONE_TAG,
-      workerpool: vWorkerpool,
-    });
-    const apporder = appOrderbook.orders[0]?.order; //The first order is the cheapest one
-    if (!apporder) {
-      throw new Error(`No app orders found`);
+    if (!workerpoolorder) {
+      throw new Error('No Workerpool order found for the desired price');
     }
-    vOnStatusUpdate({
-      title: 'FETCH_APP_ORDERBOOK',
-      isDone: true,
-    });
 
-    vOnStatusUpdate({
-      title: 'FETCH_WORKERPOOL_ORDERBOOK',
-      isDone: false,
-    });
-    const workerpoolOrderbook = await iexec.orderbook.fetchWorkerpoolOrderbook({
-      workerpool: vWorkerpool === ethers.ZeroAddress ? 'any' : vWorkerpool, // if address zero was chosen use any workerpool
-      app: vApp,
-      dataset: vProtectedData,
-      requester: requester,
-      isRequesterStrict:
-        vVoucherOwner && vVoucherOwner.toLowerCase() !== requester.toLowerCase()
-          ? false
-          : useVoucher,
-      minTag: SCONE_TAG,
-      maxTag: SCONE_TAG,
-      category: 0,
-    });
-    const workerpoolOrder = findWorkerpoolOrders({
-      workerpoolOrders: [...workerpoolOrderbook.orders],
-      useVoucher: vUseVoucher,
-      userVoucher,
-    });
-    if (!workerpoolOrder) {
-      throw new Error('No Workerpool order found.');
+    const datasetorder = datasetorderForApp || datasetorderForWhitelist;
+    if (!datasetorder) {
+      throw new Error('No Dataset order found for the desired price');
     }
     vOnStatusUpdate({
-      title: 'FETCH_WORKERPOOL_ORDERBOOK',
+      title: 'FETCH_ORDERS',
       isDone: true,
     });
-
     vOnStatusUpdate({
       title: 'PUSH_REQUESTER_SECRET',
       isDone: false,
@@ -222,13 +267,13 @@ export const processProtectedData = async ({
     });
     const requestorderToSign = await iexec.order.createRequestorder({
       app: vApp,
-      category: workerpoolOrder.category,
+      category: workerpoolorder.category,
       dataset: vProtectedData,
       appmaxprice: apporder.appprice,
       datasetmaxprice: datasetorder.datasetprice,
-      workerpoolmaxprice: workerpoolOrder.workerpoolprice,
+      workerpoolmaxprice: workerpoolorder.workerpoolprice,
       tag: SCONE_TAG,
-      workerpool: workerpoolOrder.workerpool,
+      workerpool: workerpoolorder.workerpool,
       params: {
         iexec_input_files: vInputFiles,
         iexec_secrets: secretsId,
@@ -239,7 +284,7 @@ export const processProtectedData = async ({
 
     const orders = {
       requestorder,
-      workerpoolorder: workerpoolOrder,
+      workerpoolorder: workerpoolorder,
       apporder: apporder,
       datasetorder: datasetorder,
     };
@@ -248,20 +293,6 @@ export const processProtectedData = async ({
       ...(vVoucherOwner ? { voucherAddress: userVoucher?.address } : {}),
     };
 
-    const estimatedMatchOrderPrice = await iexec.order.estimateMatchOrders(
-      orders,
-      matchOptions
-    );
-    if (
-      estimatedMatchOrderPrice.total
-        .sub(estimatedMatchOrderPrice.sponsored)
-        .ltn(vMaxPrice)
-    ) {
-      throw new Error(
-        `No orders found within the specified price limit ${vMaxPrice} nRLC.`
-      );
-    }
-
     const { dealid, txHash } = await iexec.order.matchOrders(
       orders,
       matchOptions

packages/sdk/src/lib/types/coreTypes.ts

@@ -276,6 +276,7 @@ export type TransferResponse = {
 
 // ---------------------ProcessProtectedData Types------------------------------------
 export type ProcessProtectedDataStatuses =
+  | 'FETCH_ORDERS'
   | 'FETCH_PROTECTED_DATA_ORDERBOOK'
   | 'FETCH_APP_ORDERBOOK'
   | 'FETCH_WORKERPOOL_ORDERBOOK'
@@ -302,11 +303,22 @@ export type ProcessProtectedDataParams = {
   userWhitelist?: Address;
 
   /**
-   * The maximum price per task for processing the protected data.
-   * It is the sum of the application price, dataset price and workerpool price per task.
+   * The maximum price of dataset per task for processing the protected data.
   @default = 0
   */
-  maxPrice?: number;
+  dataMaxPrice?: number;
+
+  /**
+   * The maximum price of application per task for processing the protected data.
+  @default = 0
+  */
+  appMaxPrice?: number;
+
+  /**
+   * The maximum price of workerpool per task for processing the protected data.
+  @default = 0
+  */
+  workerpoolMaxPrice?: number;
 
   /**
    * The file name of the desired file in the returned ZIP file.

packages/sdk/src/utils/processProtectedData.models.ts

@@ -24,12 +24,14 @@ export function checkUserVoucher({
   }
 }
 
-export function findWorkerpoolOrders({
+export function filterWorkerpoolOrders({
   workerpoolOrders,
+  workerpoolMaxPrice,
   useVoucher,
   userVoucher,
 }: {
   workerpoolOrders: PublishedWorkerpoolorder[];
+  workerpoolMaxPrice: number;
   useVoucher: boolean;
   userVoucher?: VoucherInfo;
 }) {
@@ -38,6 +40,8 @@ export function findWorkerpoolOrders({
   }
 
   let eligibleWorkerpoolOrders = workerpoolOrders;
+  let maxVoucherSponsoredAmount = 0; // may be safer to use bigint
+
   if (useVoucher) {
     if (!userVoucher) {
       throw new Error(
@@ -53,11 +57,20 @@ export function findWorkerpoolOrders({
         'Found some workerpool orders but none can be sponsored by your voucher.'
       );
     }
+    maxVoucherSponsoredAmount = bnToNumber(userVoucher.balance);
   }
 
   const [cheapestOrder] = eligibleWorkerpoolOrders.sort(
     (order1, order2) =>
       order1.order.workerpoolprice - order2.order.workerpoolprice
   );
+
+  if (
+    !cheapestOrder ||
+    cheapestOrder.order.workerpoolprice >
+      workerpoolMaxPrice + maxVoucherSponsoredAmount
+  ) {
+    return null;
+  }
   return cheapestOrder.order;
 }

packages/sdk/tests/test-utils.ts

@@ -211,6 +211,32 @@ export const mockWorkerpoolOrderbook = {
   count: 1,
 };
 
+export const mockAppOrderbook = {
+  orders: [
+    {
+      order: {
+        app: '0xc8c5E295D2BedA01D1fB8DD4d85A1Cb769185a34',
+        appprice: 0,
+        volume: 10000000,
+        tag: '0x0000000000000000000000000000000000000000000000000000000000000003',
+        datasetrestrict: '0x0000000000000000000000000000000000000000',
+        workerpoolrestrict: '0x0000000000000000000000000000000000000000',
+        requesterrestrict: '0x0000000000000000000000000000000000000000',
+        salt: '0x82107d3b5694d3ab4cd4e5f2057e1bdeb7da359518ccfb15638405c619fa12b0',
+        sign: '0x0112d6f1b53777a001054daf62f542a5f94679f88885515a2126a6794505d6993e425c3a432b4c2cdbf004f6f0c8c9908493135dedb829f6c958e67daa068dd61c',
+      },
+      orderHash:
+        '0x64208bc3580bbee092c4a4efb26629cf885a2f1e99b6b4d9bd809ea85b58332f',
+      chainId: 134,
+      publicationTimestamp: '2025-02-05T14:35:51.271Z',
+      signer: '0x9cfFa14604A6836E9d6fBAcCc624cfE0bE3Be5B4',
+      status: 'open',
+      remaining: 9999961,
+    },
+  ],
+  count: 1,
+};
+
 export function observableMockComplete() {
   const mockObservable: any = {
     subscribe: jest.fn(({ complete }) => {