feat: add support for dataset bulk orders (#38)

* feat: add bulkOnly filters for datasetorders

* feat: add bulk metadata to datasetorders

Author: PierreJeanjacquot

api/openapi.yaml

@@ -477,6 +477,11 @@ paths:
           description: exclude orders with permissions allowing "any" requester (default false, ignored if `requester` is not specified)
           schema:
             type: boolean
+        - name: bulkOnly
+          in: query
+          description: exclude orders not eligible to bulk processing (default false)
+          schema:
+            type: boolean
         - $ref: '#/components/parameters/PageIndex'
         - $ref: '#/components/parameters/PageSize'
         - $ref: '#/components/parameters/Page'
@@ -1130,6 +1135,12 @@ components:
         remaining:
           type: number
 
+    OrderBulkMetadata:
+      properties:
+        bulk:
+          type: boolean
+          example: false
+
     AppOrder:
       properties:
         app:
@@ -1243,6 +1254,7 @@ components:
     DatasetOrderWithMetadata:
       allOf:
         - $ref: '#/components/schemas/OrderMetadata'
+        - $ref: '#/components/schemas/OrderBulkMetadata'
         - type: object
           properties:
             order:

api/src/controllers/router.js

@@ -373,6 +373,7 @@ router.get('/datasetorders', bodyParser(), async (ctx) => {
     minTag,
     maxTag,
     minVolume,
+    bulkOnly,
     page,
     pageIndex,
     pageSize,
@@ -394,6 +395,7 @@ router.get('/datasetorders', bodyParser(), async (ctx) => {
     minTag: bytes32Schema(),
     maxTag: bytes32Schema(),
     minVolume: positiveStrictIntSchema(),
+    bulkOnly: booleanSchema().default(false),
     page: positiveIntSchema(),
     pageIndex: positiveIntSchema(),
     pageSize: positiveIntSchema().max(maxPageSize).min(minPageSize),
@@ -411,6 +413,7 @@ router.get('/datasetorders', bodyParser(), async (ctx) => {
     minTag,
     maxTag,
     minVolume,
+    bulkOnly,
     page,
     pageIndex,
     pageSize,

api/src/models/datasetorderModel.js

@@ -2,6 +2,7 @@ import { Schema } from 'mongoose';
 import { getMongoose } from '../loaders/mongoose.js';
 import { logger } from '../utils/logger.js';
 import { option, schema } from './common.js';
+import { isDatasetBulkOrder } from '../utils/order-utils.js';
 
 const {
   AddressSchema,
@@ -20,6 +21,16 @@ const log = logger.extend('models:datasetorderModel');
 
 const connectedModels = {};
 
+const toJSON = {
+  ...orderToJsonOption.toJSON,
+  transform(doc, ret) {
+    // Apply base transform
+    orderToJsonOption.toJSON.transform(doc, ret);
+    // Add bulk field
+    ret.bulk = isDatasetBulkOrder(ret?.order);
+  },
+};
+
 const datasetorderSchema = new Schema(
   {
     orderHash: { ...Bytes32Schema, unique: true, index: true },
@@ -42,7 +53,7 @@ const datasetorderSchema = new Schema(
     signer: { ...AddressSchema, index: true },
   },
   {
-    ...orderToJsonOption,
+    toJSON,
   },
 );
 

api/src/services/order.js

@@ -576,6 +576,7 @@ const getDatasetorders = async ({
   minTag,
   maxTag,
   minVolume,
+  bulkOnly,
   page,
   pageIndex,
   pageSize,
@@ -586,6 +587,10 @@ const getDatasetorders = async ({
       status: STATUS_MAP.OPEN,
       ...(dataset && requiredDatasetOrAnyClause(dataset)),
       ...(datasetOwner && { signer: datasetOwner }),
+      ...(bulkOnly && {
+        'order.volume': Number.MAX_SAFE_INTEGER,
+        'order.datasetprice': 0,
+      }),
       ...apprestrictOrAnyClause(app, isAppStrict),
       ...workerpoolrestrictOrAnyClause(workerpool, isWorkerpoolStrict),
       ...requesterrestrictOrAnyClause(requester, isRequesterStrict),

api/src/utils/order-utils.js

@@ -139,11 +139,16 @@ const excludeTagArray = (tagArray) =>
     })
     .filter((e) => e !== null);
 
+const isDatasetBulkOrder = (order = {}) => {
+  return order?.volume >= Number.MAX_SAFE_INTEGER && order?.datasetprice === 0;
+};
+
 export {
   OBJ_MAP,
   STATUS_MAP,
   TAG_MAP,
   UNPUBLISH_TARGET_MAP,
   tagToArray,
   excludeTagArray,
+  isDatasetBulkOrder,
 };

api/test/datasetorders.test.js

@@ -904,6 +904,7 @@ describe('Offchain marketplace', () => {
     const maxGpuTagOrders = [];
     const minMaxTeeTagOrders = [];
     const minVolumeOrders = [];
+    const bulkOrders = [];
     let consumedOrders;
     let deadOrders;
     let datasetAddress;
@@ -949,6 +950,30 @@ describe('Offchain marketplace', () => {
       noRestrictOrders.push(...datasetPrice0);
       allOrders.push(...datasetPrice0);
 
+      const bulk = await Promise.all(
+        Array(2)
+          .fill(null)
+          .map(async () => {
+            const order = await iexecUser.order
+              .createDatasetorder({
+                dataset: datasetAddress,
+                datasetprice: 0, // bulk order must be free
+                volume: Number.MAX_SAFE_INTEGER, // bulk order must have max volume
+              })
+              .then(iexecUser.order.signDatasetorder);
+            const orderHash = await iexecUser.order.hashDatasetorder(order);
+            return {
+              order,
+              orderHash,
+              signer: ownerAddress,
+            };
+          }),
+      );
+      bulkOrders.push(...bulk);
+      minVolumeOrders.push(...bulk);
+      noRestrictOrders.push(...bulk);
+      allOrders.push(...bulk);
+
       const datasetPrice20 = await Promise.all(
         Array(5)
           .fill(null)
@@ -1803,6 +1828,52 @@ describe('Offchain marketplace', () => {
       });
     });
 
+    test('GET /datasetorders (bulkOnly filter)', async () => {
+      const onlyBulkRes = await request
+        .get(
+          buildQuery('/datasetorders', {
+            chainId, // *
+            dataset: datasetAddress, // *
+            bulkOnly: true,
+          }),
+        )
+        .then(parseResult);
+      expect(onlyBulkRes.status).toBe(OK_STATUS);
+      expect(onlyBulkRes.data.ok).toBe(true);
+      expect(onlyBulkRes.data.count).toBe(bulkOrders.length);
+      expect(onlyBulkRes.data.orders).toBeDefined();
+      expect(Array.isArray(onlyBulkRes.data.orders)).toBe(true);
+      expect(onlyBulkRes.data.orders.length).toBe(bulkOrders.length);
+      onlyBulkRes.data.orders.forEach((e) => {
+        expect(e.bulk).toBe(true);
+      });
+
+      const notOnlyBulkRes = await request
+        .get(
+          buildQuery('/datasetorders', {
+            chainId, // *
+            dataset: datasetAddress, // *
+            // bulkOnly: false,
+          }),
+        )
+        .then(parseResult);
+      expect(notOnlyBulkRes.status).toBe(OK_STATUS);
+      expect(notOnlyBulkRes.data.ok).toBe(true);
+      expect(notOnlyBulkRes.data.count).toBe(publicOrders.length);
+      expect(notOnlyBulkRes.data.orders).toBeDefined();
+      expect(Array.isArray(notOnlyBulkRes.data.orders)).toBe(true);
+      notOnlyBulkRes.data.orders.forEach((e) => {
+        if (
+          e.order.volume >= Number.MAX_SAFE_INTEGER &&
+          e.order.datasetprice === 0
+        ) {
+          expect(e.bulk).toBe(true);
+        } else {
+          expect(e.bulk).toBe(false);
+        }
+      });
+    });
+
     test('GET /datasetorders (isAppStrict = true & app = undefined): should return public orders including "any" app', async () => {
       const result = await request
         .get(