feat: add bulk processing in python template

Author: PierreJeanjacquot

cli/src/utils/initIAppWorkspace.ts

@@ -150,6 +150,12 @@ async function copyChosenTemplateFiles({
         ''
       );
     }
+    if (!useBulkProcessing && !useProtectedData) {
+      modifiedCode = modifiedCode.replaceAll(
+        / *(\/\/|#) <<protectedData\|bulkProcessing>>\n((.*)\n)*? *(\/\/|#) <<\/protectedData\|bulkProcessing>>(\n)?/g,
+        ''
+      );
+    }
     if (!useInputFile) {
       modifiedCode = modifiedCode.replaceAll(
         / *(\/\/|#) <<inputFile>>\n((.*)\n)*? *(\/\/|#) <<\/inputFile>>(\n)?/g,

cli/templates/JavaScript/package.json

@@ -5,9 +5,9 @@
   "main": "src/app.js",
   "type": "module",
   "dependencies": {
-    // <<protectedData>>
+    // <<protectedData|bulkProcessing>>
     "@iexec/dataprotector-deserializer": "^0.1.1",
-    // <</protectedData>>
+    // <</protectedData|bulkProcessing>>
     "figlet": "^1.8.1"
   }
 }

cli/templates/JavaScript/src/app.js

@@ -3,12 +3,9 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 // <</bulkProcessing>>
 import figlet from 'figlet';
-// <<protectedData>>
+// <<protectedData|bulkProcessing>>
 import { IExecDataProtectorDeserializer } from '@iexec/dataprotector-deserializer';
-// <</protectedData>>
-// <<bulkProcessing>>
-import { IExecDataProtectorDeserializer } from '@iexec/dataprotector-deserializer';
-// <</bulkProcessing>>
+// <</protectedData|bulkProcessing>>
 
 const main = async () => {
   const { IEXEC_OUT } = process.env;
@@ -31,9 +28,9 @@ const main = async () => {
       const deserializer = new IExecDataProtectorDeserializer();
       // The protected data mock created for the purpose of this Hello World journey
       // contains an object with a key "secretText" which is a string
-      const protectedName = await deserializer.getValue('secretText', 'string');
+      const protectedText = await deserializer.getValue('secretText', 'string');
       console.log('Found a protected data');
-      messages.push(protectedName);
+      messages.push(protectedText);
     } catch (e) {
       console.log('It seems there is an issue with your protected data:', e);
     }
@@ -53,12 +50,12 @@ const main = async () => {
           });
           // The protected data mock created for the purpose of this Hello World journey
           // contains an object with a key "secretText" which is a string
-          const protectedName = await deserializer.getValue(
+          const protectedText = await deserializer.getValue(
             'secretText',
             'string'
           );
           console.log(`Found protected data ${i} of bulk`);
-          messages.push(protectedName);
+          messages.push(protectedText);
         } catch (e) {
           console.log(
             `It seems there is an issue with protected data ${i}:`,

cli/templates/Python3.13/requirements.txt

@@ -1,5 +1,5 @@
 # requirements must be compatible with python 3.13.3
 pyfiglet==1.0.2
-# <<protectedData>>
+# <<protectedData|bulkProcessing>>
 borsh-construct==0.1.0
-# <</protectedData>>
+# <</protectedData|bulkProcessing>>

cli/templates/Python3.13/src/app.py

@@ -7,9 +7,9 @@
 import shutil
 # <</inputFile>>
 from pyfiglet import Figlet
-# <<protectedData>>
+# <<protectedData|bulkProcessing>>
 import protected_data
-# <</protectedData>>
+# <</protectedData|bulkProcessing>>
 
 # ⚠️ Your Python code will be run in a python v3.8.3 environment
 
@@ -32,10 +32,28 @@
         # The protected data mock created for the purpose of this Hello World journey
         # contains an object with a key "secretText" which is a string
         protected_text = protected_data.getValue('secretText', 'string')
+        print('Found a protected data')
         messages.append(protected_text)
     except Exception as e:
         print('It seems there is an issue with your protected data:', e)
     # <</protectedData>>
+    # <<bulkProcessing>>
+
+    bulkSize = int(os.getenv("IEXEC_BULK_SLICE_SIZE", "0"))
+    if bulkSize > 0:
+        print(f"Got {bulkSize} protected data to process in bulk!")
+        for i in range(1, bulkSize + 1):
+            try:
+                # The protected data mock created for the purpose of this Hello World journey
+                # contains an object with a key "secretText" which is a string
+                protected_text = protected_data.getValue(
+                    'secretText', 'string', i)
+                print(f"Found protected data {i} of bulk")
+                messages.append(protected_text)
+            except Exception as e:
+                print(
+                    f"It seems there is an issue with your protected data {i}:", e)
+    # <</bulkProcessing>>
     # <<inputFile>>
 
     IEXEC_INPUT_FILES_NUMBER = int(os.getenv("IEXEC_INPUT_FILES_NUMBER", 0))

cli/templates/Python3.13/src/protected_data.py

@@ -1,19 +1,25 @@
-# <<protectedData>>
+# <<protectedData|bulkProcessing>>
 # dataprotector deserializer module
 import os
 import zipfile
 from borsh_construct import String, I128, F64, Bool
 
 
-def getValue(path: str, schema: str):
+def getValue(path: str, schema: str, bulkIndex: int = None) -> any:
     file_path = path.replace('.', '/')
     IEXEC_IN = os.getenv('IEXEC_IN')
-    IEXEC_DATASET_FILENAME = os.getenv('IEXEC_DATASET_FILENAME')
+    file_name: str
 
-    if IEXEC_DATASET_FILENAME == None:
+    if bulkIndex != None:
+        file_name = os.getenv(
+            f'IEXEC_DATASET_{bulkIndex}_FILENAME')
+    else:
+        file_name = os.getenv('IEXEC_DATASET_FILENAME')
+
+    if file_name == None:
         raise Exception('Missing protected data')
 
-    dataset_file_path = os.path.join(IEXEC_IN, IEXEC_DATASET_FILENAME)
+    dataset_file_path = os.path.join(IEXEC_IN, file_name)
 
     file_bytes: bytes
     try:
@@ -38,4 +44,4 @@ def getValue(path: str, schema: str):
         raise Exception(f"Failed to deserialize \"{path}\" as \"{schema}\"")
 
     return file_bytes
-# <</protectedData>>
+# <</protectedData|bulkProcessing>>