Merge branch 'main' into fix_extract_image_block_type

Author: jordan-homan

package-lock.json

@@ -20,6 +20,7 @@
   "devDependencies": {
     "@types/async": "^3.2.24",
     "@types/jest": "^29.5.12",
+    "@types/uuid": "^10.0.0",
     "@typescript-eslint/eslint-plugin": "^7.7.1",
     "@typescript-eslint/parser": "^7.7.1",
     "eslint": "^8.57.0",
@@ -32,6 +33,7 @@
   },
   "dependencies": {
     "async": "^3.2.5",
-    "pdf-lib": "^1.17.1"
+    "pdf-lib": "^1.17.1",
+    "uuid": "^10.0.0"
   }
 }

package.json

@@ -1,4 +1,5 @@
 import async from "async";
+import { v4 as uuidv4 } from 'uuid';
 
 import {
   AfterErrorContext,
@@ -96,7 +97,11 @@ export class SplitPdfHook
     hookCtx: BeforeRequestContext,
     request: Request
   ): Promise<Request> {
-    const { operationID } = hookCtx;
+
+    // setting the current operationID to be unique
+    const operationID = "partition-" + uuidv4();
+    hookCtx.operationID = operationID;
+
     const requestClone = request.clone();
     const formData = await requestClone.formData();
     const splitPdfPage = stringToBoolean(

src/hooks/custom/SplitPdfHook.ts

@@ -393,4 +393,78 @@ describe("SplitPDF succeeds for large PDF with high concurrency", () => {
         expect(res.elements?.length).toBeGreaterThan(0);
     },
     300000);
-});
+});
+
+
+describe("SplitPDF async can be used to send multiple files concurrently", () => {
+  const FAKE_API_KEY = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+
+  it.each([
+    `${localServer}/general/v0/general`,
+  ])("succeed", async (serverURL) => {
+      const client = new UnstructuredClient({
+          serverURL: serverURL,
+          security: {
+              apiKeyAuth: FAKE_API_KEY,
+          },
+      });
+
+      const file = {
+          content: readFileSync("test/data/layout-parser-paper.pdf"),
+          fileName: "test/data/layout-parser-paper.pdf"
+      };
+
+      const RequestsParams = [
+        {
+            files: file,
+            splitPdfPage: true,
+            strategy: Strategy.Fast,
+            splitPdfPageRange: [1, 3],
+            languages: ["eng"],
+        },
+        {
+            files: file,
+            splitPdfPage: true,
+            strategy: Strategy.Fast,
+            splitPdfPageRange: [10, 12],
+            languages: ["eng"],
+        }
+      ];
+
+      // Process requests serially
+      const serialElements: any[][] = [];
+      for (const requestParams of RequestsParams) {
+        const res: PartitionResponse = await client.general.partition({
+          partitionParameters: {
+            ...requestParams
+          },
+        });
+        expect(res.statusCode).toEqual(200);
+        expect(res.elements?.length).toBeGreaterThan(0);
+        if (res.elements) {
+          serialElements.push(res.elements);
+        }
+      }
+
+      // Process requests concurrently
+      const concurrentElements: any[][] = [];
+      const concurrentResponses = await Promise.all(RequestsParams.map(req =>
+        client.general.partition({
+            partitionParameters: req
+        })
+      ));
+
+      for (const res of concurrentResponses) {
+        expect(res.statusCode).toEqual(200);
+        expect(res.elements?.length).toBeGreaterThan(0);
+        if (res.elements) {
+          concurrentElements.push(res.elements);
+        }
+      }
+
+      const isEqual = JSON.stringify(serialElements) === JSON.stringify(concurrentElements);
+      expect(isEqual).toBe(true);
+
+  },
+  300000);
+});