feat(sdk): add arweave uploadMode option (#448)

Author: PierreJeanjacquot

packages/sdk/package-lock.json

@@ -73,6 +73,7 @@
     "@swc/core": "^1.7.3",
     "@swc/jest": "^0.2.36",
     "@types/debug": "^4.1.7",
+    "@types/express": "^5.0.2",
     "@types/jest": "^29.5.12",
     "@typescript-eslint/eslint-plugin": "^6.7.5",
     "@typescript-eslint/parser": "^6.7.5",
@@ -85,6 +86,7 @@
     "eslint-plugin-jest": "^27.4.2",
     "eslint-plugin-sonarjs": "^0.21.0",
     "eslint-plugin-unicorn": "^51.0.1",
+    "express": "^5.1.0",
     "jest": "^29.7.0",
     "prettier": "^2.8.4",
     "rimraf": "^5.0.5",

packages/sdk/package.json

@@ -27,6 +27,9 @@ export const CHAIN_CONFIG: Record<ChainId, ChainConfig> = {
 };
 
 export const DEFAULT_CHAIN_ID = 134;
+export const DEFAULT_ARWEAVE_UPLOAD_API = 'https://arweave-api.iex.ec';
+export const DEFAULT_ARWEAVE_GATEWAY = 'https://arweave.net';
+export const ARWEAVE_FREE_UPLOAD_MAX_SIZE = 100 * 1024; // 100kb
 export const DEFAULT_DATA_NAME = '';
 export const SCONE_TAG = ['tee', 'scone'];
 export const DEFAULT_MAX_PRICE = 0;

packages/sdk/src/config/config.ts

@@ -1,7 +1,7 @@
 import { AbstractProvider, AbstractSigner, Eip1193Provider } from 'ethers';
 import { GraphQLClient } from 'graphql-request';
 import { IExec } from 'iexec';
-import { CHAIN_CONFIG } from '../config/config.js';
+import { CHAIN_CONFIG, DEFAULT_ARWEAVE_UPLOAD_API } from '../config/config.js';
 import { getChainIdFromProvider } from '../utils/getChainId.js';
 import {
   AddressOrENS,
@@ -38,6 +38,8 @@ abstract class IExecDataProtectorModule {
 
   protected ipfsGateway!: string;
 
+  protected arweaveUploadApi: string = DEFAULT_ARWEAVE_UPLOAD_API;
+
   protected defaultWorkerpool!: string;
 
   protected iexec!: IExec;

packages/sdk/src/lib/IExecDataProtectorModule.ts

@@ -39,6 +39,7 @@ class IExecDataProtectorCore extends IExecDataProtectorModule {
       dataprotectorContractAddress: this.dataprotectorContractAddress,
       ipfsNode: this.ipfsNode,
       ipfsGateway: this.ipfsGateway,
+      arweaveUploadApi: this.arweaveUploadApi,
       iexec: this.iexec,
       iexecDebug: this.iexecDebug,
     });

packages/sdk/src/lib/dataProtectorCore/IExecDataProtectorCore.ts

@@ -1,6 +1,10 @@
 import { multiaddr as Multiaddr } from '@multiformats/multiaddr';
-import { DEFAULT_DATA_NAME } from '../../config/config.js';
-import { add } from '../../services/ipfs.js';
+import {
+  DEFAULT_ARWEAVE_GATEWAY,
+  DEFAULT_DATA_NAME,
+} from '../../config/config.js';
+import * as arweave from '../../services/arweave.js';
+import * as ipfs from '../../services/ipfs.js';
 import {
   createZipFromObject,
   ensureDataObjectIsValid,
@@ -28,6 +32,7 @@ import {
   ProtectedDataWithSecretProps,
 } from '../types/index.js';
 import {
+  ArweaveUploadConsumer,
   DataProtectorContractConsumer,
   IExecConsumer,
   IExecDebugConsumer,
@@ -43,17 +48,24 @@ export const protectData = async ({
   iexecDebug = throwIfMissing(),
   dataprotectorContractAddress,
   name = DEFAULT_DATA_NAME,
+  uploadMode = 'ipfs',
   ipfsNode,
   ipfsGateway,
+  arweaveUploadApi,
   allowDebug = false,
   data,
   onStatusUpdate = () => {},
 }: IExecConsumer &
   IExecDebugConsumer &
   DataProtectorContractConsumer &
   IpfsNodeAndGateway &
+  ArweaveUploadConsumer &
   ProtectDataParams): Promise<ProtectedDataWithSecretProps> => {
   const vName = stringSchema().label('name').validateSync(name);
+  const vUploadMode = stringSchema()
+    .oneOf(['ipfs', 'arweave'])
+    .label('uploadMode')
+    .validateSync(uploadMode);
   const vIpfsNodeUrl = urlSchema().label('ipfsNode').validateSync(ipfsNode);
   const vIpfsGateway = urlSchema()
     .label('ipfsGateway')
@@ -149,28 +161,55 @@ export const protectData = async ({
       title: 'UPLOAD_ENCRYPTED_FILE',
       isDone: false,
     });
-    const cid = await add(encryptedFile, {
-      ipfsNode: vIpfsNodeUrl,
-      ipfsGateway: vIpfsGateway,
-    }).catch((e: Error) => {
-      throw new WorkflowError({
-        message: 'Failed to upload encrypted data',
-        errorCause: e,
+
+    let multiaddr: string;
+    let multiaddrBytes: Uint8Array;
+
+    if (vUploadMode === 'arweave') {
+      const arweaveId = await arweave
+        .add(encryptedFile, { arweaveUploadApi })
+        .catch((e: Error) => {
+          throw new WorkflowError({
+            message: 'Failed to upload encrypted data',
+            errorCause: e,
+          });
+        });
+      multiaddr = `${DEFAULT_ARWEAVE_GATEWAY}/${arweaveId}`;
+      multiaddrBytes = new TextEncoder().encode(multiaddr);
+      vOnStatusUpdate({
+        title: 'UPLOAD_ENCRYPTED_FILE',
+        isDone: true,
+        payload: {
+          arweaveId,
+        },
       });
-    });
-    const multiaddr = `/p2p/${cid}`;
-    vOnStatusUpdate({
-      title: 'UPLOAD_ENCRYPTED_FILE',
-      isDone: true,
-      payload: {
-        cid,
-      },
-    });
+    } else {
+      // ipfs fallback
+      const cid = await ipfs
+        .add(encryptedFile, {
+          ipfsNode: vIpfsNodeUrl,
+          ipfsGateway: vIpfsGateway,
+        })
+        .catch((e: Error) => {
+          throw new WorkflowError({
+            message: 'Failed to upload encrypted data',
+            errorCause: e,
+          });
+        });
+      multiaddr = `/p2p/${cid}`;
+      multiaddrBytes = Multiaddr(multiaddr).bytes;
+      vOnStatusUpdate({
+        title: 'UPLOAD_ENCRYPTED_FILE',
+        isDone: true,
+        payload: {
+          cid,
+        },
+      });
+    }
 
     const { provider, signer, txOptions } =
       await iexec.config.resolveContractsClient();
 
-    const multiaddrBytes = Multiaddr(multiaddr).bytes;
     const ownerAddress = await signer.getAddress();
 
     vOnStatusUpdate({

packages/sdk/src/lib/dataProtectorCore/protectData.ts

@@ -72,6 +72,14 @@ export type ProtectDataParams = {
    */
   allowDebug?: boolean;
 
+  /**
+   * specify the platform used for storing the encrypted payload of the protected data
+   *
+   * - `"ipfs"` (default): https://ipfs.tech/
+   * - `"arweave"`: https://arweave.org/
+   */
+  uploadMode?: 'ipfs' | 'arweave';
+
   /**
    * Callback function that will get called at each step of the process
    */
@@ -98,6 +106,7 @@ export type ProtectedDataCreationProps = {
   transactionHash: string;
   zipFile: Uint8Array;
   encryptionKey: string;
+  multiaddr: string;
 };
 
 export type ProtectedDataWithSecretProps = ProtectedData &

packages/sdk/src/lib/types/coreTypes.ts

@@ -17,6 +17,10 @@ export type DataProtectorContractConsumer = {
   dataprotectorContractAddress: AddressOrENS;
 };
 
+export type ArweaveUploadConsumer = {
+  arweaveUploadApi?: string;
+};
+
 export type SubgraphConsumer = {
   graphQLClient: GraphQLClient;
 };

packages/sdk/src/lib/types/internalTypes.ts

@@ -0,0 +1,66 @@
+import {
+  DEFAULT_ARWEAVE_GATEWAY,
+  DEFAULT_ARWEAVE_UPLOAD_API,
+  ARWEAVE_FREE_UPLOAD_MAX_SIZE,
+} from '../config/config.js';
+
+interface AddOptions {
+  arweaveUploadApi?: string;
+  arweaveGateway?: string;
+}
+
+const add = async (
+  content: Uint8Array,
+  { arweaveGateway, arweaveUploadApi }: AddOptions = {}
+): Promise<string> => {
+  if (content.length >= ARWEAVE_FREE_UPLOAD_MAX_SIZE) {
+    throw Error(
+      `Arweave upload ${(ARWEAVE_FREE_UPLOAD_MAX_SIZE / 1024).toFixed(
+        0
+      )}kb size limit reached`
+    );
+  }
+
+  let arweaveId: string;
+  try {
+    const payload = new FormData();
+    payload.append('file', new Blob([content]));
+    const res = await fetch(
+      `${arweaveUploadApi || DEFAULT_ARWEAVE_UPLOAD_API}/upload`,
+      {
+        method: 'POST',
+        body: payload,
+      }
+    );
+    if (!res.ok) {
+      throw Error(`Arweave upload API answered with status ${res.status}`);
+    }
+    try {
+      const data = await res.json();
+      arweaveId = data?.arweaveId;
+      if (!arweaveId) {
+        throw Error('missing arweaveId');
+      }
+    } catch (e) {
+      throw Error('Arweave upload API answered with an invalid response', {
+        cause: e,
+      });
+    }
+  } catch (e) {
+    throw Error('Failed to add file on Arweave', { cause: e });
+  }
+  const publicUrl = `${arweaveGateway || DEFAULT_ARWEAVE_GATEWAY}/${arweaveId}`;
+  await fetch(publicUrl)
+    .then((res) => {
+      if (!res.ok) {
+        throw Error(`Arweave gateway answered with status ${res.status}`);
+      }
+    })
+    .catch((e) => {
+      throw Error(`Failed to load uploaded file at ${publicUrl}`, { cause: e });
+    });
+
+  return arweaveId;
+};
+
+export { add };

packages/sdk/src/services/arweave.ts

@@ -22,15 +22,28 @@ export function getMultiaddrAsString({
   if (!multiAddrAsBytesArray) {
     return undefined;
   }
+  const multiaddrAsBytes = new Uint8Array(multiAddrAsBytesArray);
+
+  // try utf8 encoded url
+  try {
+    const decodedString = new TextDecoder().decode(multiaddrAsBytes);
+    if (decodedString.startsWith('https://')) {
+      return decodedString;
+    }
+  } catch {
+    // noop
+  }
+
+  // try machine format multiaddr
   try {
-    const multiaddrAsBytes = new Uint8Array(multiAddrAsBytesArray);
     const decodedMultiaddr = multiaddr(multiaddrAsBytes);
     return decodedMultiaddr.toString();
-  } catch (err) {
-    console.warn(
-      `[getMultiaddrAsString] ERROR: "${multiaddrAsHexString}" is not a valid hex input string?`,
-      err
-    );
-    return undefined;
+  } catch {
+    // noop
   }
+
+  console.warn(
+    `[getMultiaddrAsString] ERROR: "${multiaddrAsHexString}" is not a valid hex encoded multiaddr?`
+  );
+  return undefined;
 }