feat(web3mail): add bulk email processing support

- Add bulk processing capabilities to sendEmail function
- Update subgraphQuery to support batch operations
- Add new sendEmailBulk test suite
- Update fetchMyContacts and fetchUserContacts for bulk support
- Extend internal types and types definitions for bulk operations
- Update package dependencies

Author: abbesBenayache

package-lock.json

@@ -49,15 +49,15 @@
   "dependencies": {
     "@ethersproject/bytes": "^5.7.0",
     "@ethersproject/random": "^5.7.0",
+    "@iexec/dataprotector": "2.0.0-beta.20-feat-add-bulk-processing-support-094df1e",
     "buffer": "^6.0.3",
     "ethers": "^6.13.2",
     "graphql-request": "^6.1.0",
-    "iexec": "^8.20.0",
+    "iexec": "8.20.0-feat-bulk-processing-62e7c5f",
     "kubo-rpc-client": "^4.1.1",
     "yup": "^1.1.1"
   },
   "devDependencies": {
-    "@iexec/dataprotector": "^2.0.0-beta.19",
     "@jest/globals": "^29.7.0",
     "@swc/core": "^1.3.96",
     "@swc/jest": "^0.2.29",

package.json

@@ -71,20 +71,24 @@ export const getValidContact = async (
     );
 
     // Convert protectedData[] into Contact[] using the map for constant time lookups
-    return protectedDataList.map(({ id, name }) => {
-      const contact = contactsMap.get(id);
-      if (contact) {
-        return {
-          address: id,
-          name: name,
-          remainingAccess: contact.remainingAccess,
-          accessPrice: contact.accessPrice,
-          owner: contact.owner,
-          accessGrantTimestamp: contact.accessGrantTimestamp,
-          isUserStrict: contact.isUserStrict,
-        };
-      }
-    });
+    return protectedDataList
+      .map(({ id, name }) => {
+        const contact = contactsMap.get(id);
+        if (contact) {
+          return {
+            address: id,
+            name: name,
+            remainingAccess: contact.remainingAccess,
+            accessPrice: contact.accessPrice,
+            owner: contact.owner,
+            accessGrantTimestamp: contact.accessGrantTimestamp,
+            isUserStrict: contact.isUserStrict,
+            grantedAccess: contact.grantedAccess,
+          };
+        }
+        return null;
+      })
+      .filter((contact) => contact !== null && contact !== undefined);
   } catch (error) {
     throw new WorkflowError({
       message: 'Failed to fetch subgraph',

src/utils/subgraphQuery.ts

@@ -1,5 +1,9 @@
 import { AbstractProvider, AbstractSigner, Eip1193Provider } from 'ethers';
 import { IExec } from 'iexec';
+import {
+  IExecDataProtectorCore,
+  ProcessBulkRequestResponse,
+} from '@iexec/dataprotector';
 import { GraphQLClient } from 'graphql-request';
 import { fetchUserContacts } from './fetchUserContacts.js';
 import { fetchMyContacts } from './fetchMyContacts.js';
@@ -10,7 +14,7 @@ import {
   SendEmailParams,
   AddressOrENS,
   Web3MailConfigOptions,
-  SendEmailResponse,
+  SendEmailSingleResponse,
   Web3SignerProvider,
   FetchMyContactsParams,
 } from './types.js';
@@ -34,6 +38,7 @@ interface Web3mailResolvedConfig {
   ipfsGateway: string;
   defaultWorkerpool: string;
   iexec: IExec;
+  dataProtector: IExecDataProtectorCore;
 }
 
 export class IExecWeb3mail {
@@ -51,6 +56,8 @@ export class IExecWeb3mail {
 
   private iexec!: IExec;
 
+  private dataProtector!: IExecDataProtectorCore;
+
   private initPromise: Promise<void> | null = null;
 
   private ethProvider: EthersCompatibleProvider;
@@ -75,6 +82,7 @@ export class IExecWeb3mail {
         this.ipfsGateway = config.ipfsGateway;
         this.defaultWorkerpool = config.defaultWorkerpool;
         this.iexec = config.iexec;
+        this.dataProtector = config.dataProtector;
       });
     }
     return this.initPromise;
@@ -105,14 +113,17 @@ export class IExecWeb3mail {
     });
   }
 
-  async sendEmail(args: SendEmailParams): Promise<SendEmailResponse> {
+  async sendEmail(
+    args: SendEmailParams
+  ): Promise<ProcessBulkRequestResponse | SendEmailSingleResponse> {
     await this.init();
     await isValidProvider(this.iexec);
     return sendEmail({
       ...args,
       workerpoolAddressOrEns:
         args.workerpoolAddressOrEns || this.defaultWorkerpool,
       iexec: this.iexec,
+      dataProtector: this.dataProtector,
       ipfsNode: this.ipfsNode,
       ipfsGateway: this.ipfsGateway,
       dappAddressOrENS: this.dappAddressOrENS,
@@ -184,6 +195,17 @@ export class IExecWeb3mail {
       throw new Error(`Failed to create GraphQLClient: ${error.message}`);
     }
 
+    const dataProtector = new IExecDataProtectorCore(this.ethProvider, {
+      iexecOptions: {
+        ipfsGatewayURL: ipfsGateway,
+        ...this.options?.iexecOptions,
+        allowExperimentalNetworks: this.options.allowExperimentalNetworks,
+      },
+      ipfsGateway,
+      ipfsNode,
+      subgraphUrl,
+    });
+
     return {
       dappAddressOrENS,
       dappWhitelistAddress: dappWhitelistAddress.toLowerCase(),
@@ -192,6 +214,7 @@ export class IExecWeb3mail {
       ipfsNode,
       ipfsGateway,
       iexec,
+      dataProtector,
     };
   }
 }

src/web3mail/IExecWeb3mail.ts

@@ -16,6 +16,7 @@ export const fetchMyContacts = async ({
   dappAddressOrENS = throwIfMissing(),
   dappWhitelistAddress = throwIfMissing(),
   isUserStrict = false,
+  bulkOnly = false,
 }: IExecConsumer &
   SubgraphConsumer &
   DappAddressConsumer &
@@ -33,5 +34,6 @@ export const fetchMyContacts = async ({
     dappWhitelistAddress,
     userAddress,
     isUserStrict: vIsUserStrict,
+    bulkOnly,
   });
 };

src/web3mail/fetchMyContacts.ts

@@ -12,7 +12,7 @@ import {
   isEnsTest,
   throwIfMissing,
 } from '../utils/validators.js';
-import { Contact, FetchUserContactsParams } from './types.js';
+import { Contact, FetchUserContactsParams, GrantedAccess } from './types.js';
 import {
   DappAddressConsumer,
   DappWhitelistAddressConsumer,
@@ -27,6 +27,7 @@ export const fetchUserContacts = async ({
   dappWhitelistAddress = throwIfMissing(),
   userAddress,
   isUserStrict = false,
+  bulkOnly = false,
 }: IExecConsumer &
   SubgraphConsumer &
   DappAddressConsumer &
@@ -47,6 +48,7 @@ export const fetchUserContacts = async ({
   const vIsUserStrict = booleanSchema()
     .label('isUserStrict')
     .validateSync(isUserStrict);
+  const vBulkOnly = booleanSchema().label('bulkOnly').validateSync(bulkOnly);
 
   try {
     const [dappOrders, whitelistOrders] = await Promise.all([
@@ -55,15 +57,16 @@ export const fetchUserContacts = async ({
         userAddress: vUserAddress,
         appAddress: vDappAddressOrENS,
         isUserStrict: vIsUserStrict,
+        bulkOnly: vBulkOnly,
       }),
       fetchAllOrdersByApp({
         iexec,
         userAddress: vUserAddress,
         appAddress: vDappWhitelistAddress,
         isUserStrict: vIsUserStrict,
+        bulkOnly: vBulkOnly,
       }),
     ]);
-
     const orders = dappOrders.concat(whitelistOrders);
     const myContacts: Omit<Contact, 'name'>[] = [];
     let web3DappResolvedAddress = vDappAddressOrENS;
@@ -77,13 +80,25 @@ export const fetchUserContacts = async ({
         order.order.apprestrict.toLowerCase() ===
           vDappWhitelistAddress.toLowerCase()
       ) {
-        const contact = {
+        const contact: Contact = {
           address: order.order.dataset.toLowerCase(),
           owner: order.signer.toLowerCase(),
           remainingAccess: order.remaining,
           accessPrice: order.order.datasetprice,
           accessGrantTimestamp: order.publicationTimestamp,
           isUserStrict: order.order.requesterrestrict !== ZeroAddress,
+          grantedAccess: {
+            dataset: order.order.dataset,
+            datasetprice: order.order.datasetprice.toString(),
+            volume: order.order.volume.toString(),
+            tag: order.order.tag,
+            apprestrict: order.order.apprestrict,
+            workerpoolrestrict: order.order.workerpoolrestrict,
+            requesterrestrict: order.order.requesterrestrict,
+            salt: order.order.salt,
+            sign: order.order.sign,
+            remainingAccess: order.remaining,
+          } as GrantedAccess,
         };
         myContacts.push(contact);
       }
@@ -94,7 +109,6 @@ export const fetchUserContacts = async ({
     return await getValidContact(graphQLClient, myContacts);
   } catch (error) {
     handleIfProtocolError(error);
-
     throw new WorkflowError({
       message: 'Failed to fetch user contacts',
       errorCause: error,
@@ -107,23 +121,24 @@ async function fetchAllOrdersByApp({
   userAddress,
   appAddress,
   isUserStrict,
+  bulkOnly,
 }: {
   iexec: IExec;
   userAddress: string;
   appAddress: string;
   isUserStrict: boolean;
+  bulkOnly?: boolean;
 }): Promise<PublishedDatasetorder[]> {
-  const ordersFirstPage = iexec.orderbook.fetchDatasetOrderbook(
-    ANY_DATASET_ADDRESS,
-    {
-      app: appAddress,
-      requester: userAddress,
-      isAppStrict: true,
-      isRequesterStrict: isUserStrict,
-      // Use maxPageSize here to avoid too many round-trips (we want everything anyway)
-      pageSize: 1000,
-    }
-  );
+  const ordersFirstPage = iexec.orderbook.fetchDatasetOrderbook({
+    dataset: ANY_DATASET_ADDRESS,
+    app: appAddress,
+    requester: userAddress,
+    isAppStrict: true,
+    isRequesterStrict: isUserStrict,
+    bulkOnly,
+    // Use maxPageSize here to avoid too many round-trips (we want everything anyway)
+    pageSize: 1000,
+  });
   const { orders: allOrders } = await autoPaginateRequest({
     request: ordersFirstPage,
   });

src/web3mail/fetchUserContacts.ts

@@ -1,6 +1,7 @@
 import { IExec } from 'iexec';
 import { AddressOrENS } from './types.js';
 import { GraphQLClient } from 'graphql-request';
+import { IExecDataProtectorCore } from '@iexec/dataprotector';
 
 export type ProtectedDataQuery = {
   id: string;
@@ -31,6 +32,8 @@ export type IExecConsumer = {
   iexec: IExec;
 };
 
+export type DataProtectorConsumer = { dataProtector: IExecDataProtectorCore };
+
 export type SubgraphConsumer = {
   graphQLClient: GraphQLClient;
 };