fix: avoid rate limiting by process data sequentially with 1sec throttling

Author: PierreJeanjacquot

dapp/src/executeTask.js

@@ -145,15 +145,22 @@ async function start() {
     // Process multiple protected data
     if (bulkSize > 0) {
       console.log(`Processing ${bulkSize} protected data...`);
-      const processPromises = new Array(bulkSize).fill(null).map((_, index) =>
-        processProtectedData({
-          index: index + 1,
+      const results = [];
+      // Process each protected data one by one to avoid rate limiting issues
+      for (let index = 1; index <= bulkSize; index += 1) {
+        // eslint-disable-next-line no-await-in-loop
+        const protectedDataResult = await processProtectedData({
+          index,
           IEXEC_IN,
           appDeveloperSecret,
           requesterSecret,
-        })
-      );
-      const results = await Promise.all(processPromises);
+        });
+        results.push(protectedDataResult);
+        // eslint-disable-next-line no-await-in-loop
+        await new Promise((res) => {
+          setTimeout(res, 1000);
+        }); // Add a delay to avoid rate limiting
+      }
       const successCount = results.filter((r) => r.success === true).length;
       const errorCount = results.filter((r) => r.success !== true).length;
       result = {

dapp/tests/e2e/app.test.js

@@ -875,26 +875,29 @@ describe('sendEmail', () => {
           await expect(start()).resolves.toBeUndefined();
           const { result, computed, files } = await readOutputs(IEXEC_OUT);
           // Verify bulk processing result structure
-          expect(result).toStrictEqual({
-            success: true,
-            totalCount: 2,
-            successCount: 2,
-            errorCount: 0,
-            results: [
-              {
-                index: 1,
-                protectedData: process.env.IEXEC_DATASET_1_FILENAME,
-                isEmailValid: true,
-                success: true,
-              },
-              {
-                index: 2,
-                protectedData: process.env.IEXEC_DATASET_2_FILENAME,
-                isEmailValid: true,
-                success: true,
-              },
-            ],
-          });
+          expect(result).toStrictEqual(
+            {
+              success: true,
+              totalCount: 2,
+              successCount: 2,
+              errorCount: 0,
+              results: [
+                {
+                  index: 1,
+                  protectedData: process.env.IEXEC_DATASET_1_FILENAME,
+                  isEmailValid: true,
+                  success: true,
+                },
+                {
+                  index: 2,
+                  protectedData: process.env.IEXEC_DATASET_2_FILENAME,
+                  isEmailValid: true,
+                  success: true,
+                },
+              ],
+            },
+            10000
+          );
           // Verify computed.json structure
           expect(computed).toStrictEqual({
             'deterministic-output-path': `${IEXEC_OUT}/result.json`,
@@ -921,7 +924,7 @@ describe('sendEmail', () => {
           });
           // Verify no extra files were created
           expect(files.length).toBe(2);
-        });
+        }, 10000);
       });
     }
   });