feat: add bulk Telegram campaigns (#83)

Co-authored-by: Pierre Jeanjacquot <[email protected]>

Author: abbesBenayache

package-lock.json

@@ -20,18 +20,18 @@
   "scripts": {
     "build": "rm -rf dist && tsc --project tsconfig.build.json",
     "check-types": "tsc --noEmit",
-    "test:prepare": "node tests/scripts/prepare-bellecour-fork-for-tests.js",
+    "test:prepare": "node tests/scripts/prepare-bellecour-fork-for-tests.js && node tests/scripts/prepare-iexec.js",
     "test": "NODE_OPTIONS=--experimental-vm-modules jest --testMatch \"**/tests/**/*.test.ts\" --forceExit -b",
     "test:coverage": "NODE_OPTIONS=--experimental-vm-modules jest --testMatch \"**/tests/**/*.test.ts\" --forceExit --coverage",
     "test:unit": "NODE_OPTIONS=--experimental-vm-modules jest --testMatch \"**/tests/unit/**/*.test.ts\" -b",
     "test:unit:coverage": "NODE_OPTIONS=--experimental-vm-modules jest --testMatch \"**/tests/unit/**/*.unit.ts\" --coverage",
     "test:e2e": "NODE_OPTIONS=--experimental-vm-modules jest --testMatch \"**/tests/e2e/**/*.test.ts\" --forceExit -b",
     "test:e2e:coverage": "NODE_OPTIONS=--experimental-vm-modules jest --testMatch \"**/tests/e2e/**/*.test.ts\" --coverage",
     "lint": "eslint .",
-    "format": "prettier --write \"src/**/*.ts\"",
-    "check-format": "prettier --check \"src/**/*.ts\"",
+    "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\"",
+    "check-format": "prettier --check \"src/**/*.ts\" \"tests/**/*.ts\"",
     "stop-test-stack": "cd tests && docker compose down --volumes --remove-orphans",
-    "start-test-stack": "cd tests && npm run stop-test-stack && node scripts/prepare-test-env.js && docker compose build  &&  docker compose up -d && node scripts/prepare-bellecour-fork-for-tests.js"
+    "start-test-stack": "cd tests && npm run stop-test-stack && node scripts/prepare-test-env.js && docker compose build  &&  docker compose up -d && npm run test:prepare"
   },
   "repository": {
     "type": "git",
@@ -49,15 +49,15 @@
   "dependencies": {
     "@ethersproject/bytes": "^5.7.0",
     "@ethersproject/random": "^5.7.0",
+    "@iexec/dataprotector": "^2.0.0-beta.21",
     "buffer": "^6.0.3",
     "ethers": "^6.8.1",
     "graphql-request": "^6.1.0",
-    "iexec": "^8.20.0",
+    "iexec": "^8.22.0",
     "kubo-rpc-client": "^4.1.3",
     "yup": "^1.1.1"
   },
   "devDependencies": {
-    "@iexec/dataprotector": "^2.0.0-beta.19",
     "@jest/globals": "^29.7.0",
     "@swc/core": "^1.3.96",
     "@swc/jest": "^0.2.29",

package.json

@@ -71,20 +71,23 @@ export const getValidContact = async (
     );
 
     // Convert protectedData[] into Contact[] using the map for constant time lookups
-    return protectedDataList.map(({ id, name }) => {
-      const contact = contactsMap.get(id);
-      if (contact) {
-        return {
-          address: id,
-          name: name,
-          remainingAccess: contact.remainingAccess,
-          accessPrice: contact.accessPrice,
-          owner: contact.owner,
-          accessGrantTimestamp: contact.accessGrantTimestamp,
-          isUserStrict: contact.isUserStrict,
-        };
-      }
-    });
+    return protectedDataList
+      .map(({ id, name }) => {
+        const contact = contactsMap.get(id);
+        if (contact) {
+          return {
+            address: id,
+            name: name,
+            remainingAccess: contact.remainingAccess,
+            accessPrice: contact.accessPrice,
+            owner: contact.owner,
+            accessGrantTimestamp: contact.accessGrantTimestamp,
+            isUserStrict: contact.isUserStrict,
+            grantedAccess: contact.grantedAccess,
+          };
+        }
+      })
+      .filter((contact) => !!contact);
   } catch (error) {
     throw new WorkflowError({
       message: 'Failed to fetch subgraph',

src/utils/subgraphQuery.ts

@@ -1,6 +1,7 @@
 import { isAddress } from 'ethers';
 import { IExec } from 'iexec';
-import { ValidationError, boolean, number, string } from 'yup';
+import { NULL_ADDRESS } from 'iexec/utils';
+import { ValidationError, boolean, number, object, string } from 'yup';
 
 export const isValidProvider = async (iexec: IExec) => {
   const client = await iexec.config.resolveContractsClient();
@@ -53,3 +54,59 @@ export const positiveNumberSchema = () =>
 
 export const booleanSchema = () =>
   boolean().strict().typeError('${path} should be a boolean');
+
+const isPositiveIntegerStringTest = (value: string) => /^\d+$/.test(value);
+
+const stringSchema = () =>
+  string().strict().typeError('${path} should be a string');
+
+const positiveIntegerStringSchema = () =>
+  string().test(
+    'is-positive-int',
+    '${path} should be a positive integer',
+    (value) => isUndefined(value) || isPositiveIntegerStringTest(value)
+  );
+
+const positiveStrictIntegerStringSchema = () =>
+  string().test(
+    'is-positive-strict-int',
+    '${path} should be a strictly positive integer',
+    (value) =>
+      isUndefined(value) ||
+      (value !== '0' && isPositiveIntegerStringTest(value))
+  );
+
+export const campaignRequestSchema = () =>
+  object({
+    app: addressSchema().required(),
+    appmaxprice: positiveIntegerStringSchema().required(),
+    workerpool: addressSchema().required(),
+    workerpoolmaxprice: positiveIntegerStringSchema().required(),
+    dataset: addressSchema().oneOf([NULL_ADDRESS]).required(),
+    datasetmaxprice: positiveIntegerStringSchema().oneOf(['0']).required(),
+    params: stringSchema()
+      .test(
+        'is-valid-bulk-params',
+        '${path} should be a valid JSON string with bulk_cid field',
+        (value) => {
+          try {
+            // eslint-disable-next-line @typescript-eslint/naming-convention
+            const { bulk_cid } = JSON.parse(value);
+            if (typeof bulk_cid === 'string') {
+              return true;
+            }
+          } catch {}
+          return false;
+        }
+      )
+      .required(),
+    requester: addressSchema().required(),
+    beneficiary: addressSchema().required(),
+    callback: addressSchema().required(),
+    category: positiveIntegerStringSchema().required(),
+    volume: positiveStrictIntegerStringSchema().required(),
+    tag: stringSchema().required(),
+    trust: positiveIntegerStringSchema().required(),
+    salt: stringSchema().required(),
+    sign: stringSchema().required(),
+  }).typeError('${path} should be a BulkRequest object');

src/utils/validators.ts

@@ -1,23 +1,30 @@
 import { AbstractProvider, AbstractSigner, Eip1193Provider } from 'ethers';
 import { IExec } from 'iexec';
+import { IExecDataProtectorCore } from '@iexec/dataprotector';
 import { GraphQLClient } from 'graphql-request';
 import { fetchUserContacts } from './fetchUserContacts.js';
 import { fetchMyContacts } from './fetchMyContacts.js';
 import { sendTelegram } from './sendTelegram.js';
+import { sendTelegramCampaign } from './sendTelegramCampaign.js';
 import {
   Contact,
   FetchUserContactsParams,
   SendTelegramParams,
   AddressOrENS,
   Web3TelegramConfigOptions,
-  SendTelegramResponse,
   Web3SignerProvider,
   FetchMyContactsParams,
+  SendTelegramResponse,
+  PrepareTelegramCampaignResponse,
+  PrepareTelegramCampaignParams,
+  SendTelegramCampaignParams,
+  SendTelegramCampaignResponse,
 } from './types.js';
 import { getChainDefaultConfig } from '../config/config.js';
 import { isValidProvider } from '../utils/validators.js';
 import { getChainIdFromProvider } from '../utils/getChainId.js';
 import { resolveDappAddressFromCompass } from '../utils/resolveDappAddressFromCompass.js';
+import { prepareTelegramCampaign } from './prepareTelegramCampaign.js';
 
 type EthersCompatibleProvider =
   | AbstractProvider
@@ -34,6 +41,7 @@ interface Web3telegramResolvedConfig {
   ipfsGateway: string;
   defaultWorkerpool: string;
   iexec: IExec;
+  dataProtector: IExecDataProtectorCore;
 }
 
 export class IExecWeb3telegram {
@@ -51,6 +59,8 @@ export class IExecWeb3telegram {
 
   private iexec!: IExec;
 
+  private dataProtector!: IExecDataProtectorCore;
+
   private initPromise: Promise<void> | null = null;
 
   private ethProvider: EthersCompatibleProvider;
@@ -75,6 +85,7 @@ export class IExecWeb3telegram {
         this.ipfsGateway = config.ipfsGateway;
         this.defaultWorkerpool = config.defaultWorkerpool;
         this.iexec = config.iexec;
+        this.dataProtector = config.dataProtector;
       });
     }
     return this.initPromise;
@@ -121,6 +132,34 @@ export class IExecWeb3telegram {
     });
   }
 
+  async sendTelegramCampaign(
+    args: SendTelegramCampaignParams
+  ): Promise<SendTelegramCampaignResponse> {
+    await this.init();
+    await isValidProvider(this.iexec);
+    return sendTelegramCampaign({
+      ...args,
+      workerpoolAddressOrEns:
+        args.workerpoolAddressOrEns || this.defaultWorkerpool,
+      dataProtector: this.dataProtector,
+    });
+  }
+
+  async prepareTelegramCampaign(
+    args: PrepareTelegramCampaignParams
+  ): Promise<PrepareTelegramCampaignResponse> {
+    await this.init();
+
+    return prepareTelegramCampaign({
+      ...args,
+      iexec: this.iexec,
+      dataProtector: this.dataProtector,
+      ipfsNode: this.ipfsNode,
+      ipfsGateway: this.ipfsGateway,
+      dappAddressOrENS: this.dappAddressOrENS,
+    });
+  }
+
   private async resolveConfig(): Promise<Web3telegramResolvedConfig> {
     const chainId = await getChainIdFromProvider(this.ethProvider);
     const chainDefaultConfig = getChainDefaultConfig(chainId, {
@@ -185,6 +224,17 @@ export class IExecWeb3telegram {
       throw new Error(`Failed to create GraphQLClient: ${error.message}`);
     }
 
+    const dataProtector = new IExecDataProtectorCore(this.ethProvider, {
+      iexecOptions: {
+        ipfsGatewayURL: ipfsGateway,
+        ...this.options?.iexecOptions,
+        allowExperimentalNetworks: this.options.allowExperimentalNetworks,
+      },
+      ipfsGateway,
+      ipfsNode,
+      subgraphUrl,
+    });
+
     return {
       dappAddressOrENS,
       dappWhitelistAddress: dappWhitelistAddress.toLowerCase(),
@@ -193,6 +243,7 @@ export class IExecWeb3telegram {
       ipfsNode,
       ipfsGateway,
       iexec,
+      dataProtector,
     };
   }
 }

src/web3telegram/IExecWeb3telegram.ts

@@ -16,6 +16,7 @@ export const fetchMyContacts = async ({
   dappAddressOrENS = throwIfMissing(),
   dappWhitelistAddress = throwIfMissing(),
   isUserStrict = false,
+  bulkOnly = false,
 }: IExecConsumer &
   SubgraphConsumer &
   DappAddressConsumer &
@@ -33,5 +34,6 @@ export const fetchMyContacts = async ({
     dappWhitelistAddress,
     userAddress,
     isUserStrict: vIsUserStrict,
+    bulkOnly,
   });
 };

src/web3telegram/fetchMyContacts.ts

@@ -10,7 +10,7 @@ import {
   isEnsTest,
   throwIfMissing,
 } from '../utils/validators.js';
-import { Contact, FetchUserContactsParams } from './types.js';
+import { Contact, FetchUserContactsParams, GrantedAccess } from './types.js';
 import { IExec } from 'iexec';
 import { PublishedDatasetorder } from 'iexec/IExecOrderbookModule';
 import {
@@ -27,6 +27,7 @@ export const fetchUserContacts = async ({
   dappWhitelistAddress = throwIfMissing(),
   userAddress,
   isUserStrict = false,
+  bulkOnly = false,
 }: IExecConsumer &
   SubgraphConsumer &
   DappAddressConsumer &
@@ -48,19 +49,22 @@ export const fetchUserContacts = async ({
     const vIsUserStrict = booleanSchema()
       .label('isUserStrict')
       .validateSync(isUserStrict);
+    const vBulkOnly = booleanSchema().label('bulkOnly').validateSync(bulkOnly);
 
     const [dappOrders, whitelistOrders] = await Promise.all([
       fetchAllOrdersByApp({
         iexec,
         userAddress: vUserAddress,
         appAddress: vDappAddressOrENS,
         isUserStrict: vIsUserStrict,
+        bulkOnly: vBulkOnly,
       }),
       fetchAllOrdersByApp({
         iexec,
         userAddress: vUserAddress,
         appAddress: vDappWhitelistAddress,
         isUserStrict: vIsUserStrict,
+        bulkOnly: vBulkOnly,
       }),
     ]);
     const orders = dappOrders.concat(whitelistOrders);
@@ -76,13 +80,25 @@ export const fetchUserContacts = async ({
         order.order.apprestrict.toLowerCase() ===
           vDappWhitelistAddress.toLowerCase()
       ) {
-        const contact = {
+        const contact: Contact = {
           address: order.order.dataset.toLowerCase(),
           owner: order.signer.toLowerCase(),
           remainingAccess: order.remaining,
           accessPrice: order.order.datasetprice,
           accessGrantTimestamp: order.publicationTimestamp,
           isUserStrict: order.order.requesterrestrict !== ZeroAddress,
+          grantedAccess: {
+            dataset: order.order.dataset,
+            datasetprice: order.order.datasetprice.toString(),
+            volume: order.order.volume.toString(),
+            tag: order.order.tag,
+            apprestrict: order.order.apprestrict,
+            workerpoolrestrict: order.order.workerpoolrestrict,
+            requesterrestrict: order.order.requesterrestrict,
+            salt: order.order.salt,
+            sign: order.order.sign,
+            remainingAccess: order.remaining,
+          } as GrantedAccess,
         };
         myContacts.push(contact);
       }
@@ -105,23 +121,24 @@ async function fetchAllOrdersByApp({
   userAddress,
   appAddress,
   isUserStrict,
+  bulkOnly,
 }: {
   iexec: IExec;
   userAddress: string;
   appAddress: string;
   isUserStrict: boolean;
+  bulkOnly?: boolean;
 }): Promise<PublishedDatasetorder[]> {
-  const ordersFirstPage = iexec.orderbook.fetchDatasetOrderbook(
-    ANY_DATASET_ADDRESS,
-    {
-      app: appAddress,
-      requester: userAddress,
-      isAppStrict: true,
-      isRequesterStrict: isUserStrict,
-      // Use maxPageSize here to avoid too many round-trips (we want everything anyway)
-      pageSize: 1000,
-    }
-  );
+  const ordersFirstPage = iexec.orderbook.fetchDatasetOrderbook({
+    dataset: ANY_DATASET_ADDRESS,
+    app: appAddress,
+    requester: userAddress,
+    isAppStrict: true,
+    isRequesterStrict: isUserStrict,
+    bulkOnly,
+    // Use maxPageSize here to avoid too many round-trips (we want everything anyway)
+    pageSize: 1000,
+  });
   const { orders: allOrders } = await autoPaginateRequest({
     request: ordersFirstPage,
   });