feat: added allow_failed parameter (#93)

This PR:
- adds `allowFailed` parameter to control the strict mode while sending
multiple pages chunks when `splitPdf` mode is used.

Author: pawel-kmiecik

docs/sdk/models/shared/partitionparameters.md

@@ -21,3 +21,12 @@ actions:
           "description": "Number of maximum concurrent requests made when splitting PDF. Ignored on backend.",
           "default": 5,
         }
+  - target: $["components"]["schemas"]["partition_parameters"]["properties"]
+    update:
+      "split_pdf_allow_failed":
+        {
+          "title": "Split Pdf Allow Failed",
+          "description": "When `split_pdf_page` is set to `True`, this parameter defines the behavior when some of the parallel requests fail. By default `split_pdf_allow_failed` is set to `False` and any failed request send to the API will make the whole process break and raise an Exception. If `split_pdf_allow_failed` is set to `True`, the errors encountered while sending parallel requests will not break the processing - the resuling list of Elements will miss the data from errored pages.",
+          "type": "boolean",
+          "default": false,
+        }

overlay_client.yaml

@@ -12,7 +12,7 @@ import {
   SDKInitOptions,
 } from "../types.js";
 import {
-  getOptimalSplitSize,
+  getOptimalSplitSize, getSplitPdfAllowFailed,
   getSplitPdfConcurrencyLevel,
   getStartingPageNumber,
   loadPdf,
@@ -40,10 +40,21 @@ export class SplitPdfHook
    */
   client: HTTPClient | undefined;
 
+
+  /**
+   * Keeps the strict-mode setting for splitPdfPage feature.
+   */
+  allowFailed: boolean | undefined;
+
   /**
    * Maps lists responses to client operation.
    */
-  partitionResponses: Record<string, Response[]> = {};
+  partitionSuccessfulResponses: Record<string, Response[]> = {};
+
+  /**
+   * Maps lists failed responses to client operation.
+   */
+  partitionFailedResponses: Record<string, Response[]> = {};
 
   /**
    * Maps parallel requests to client operation.
@@ -115,6 +126,9 @@ export class SplitPdfHook
     const concurrencyLevel = getSplitPdfConcurrencyLevel(formData);
     console.info("Concurrency level set to %d", concurrencyLevel)
 
+    this.allowFailed = getSplitPdfAllowFailed(formData);
+    console.info("Allow failed set to %s", this.allowFailed)
+
     const splitSize = await getOptimalSplitSize(pagesCount, concurrencyLevel);
     console.info("Determined optimal split size of %d pages.", splitSize)
 
@@ -168,19 +182,30 @@ export class SplitPdfHook
       setIndex+=1;
     }
 
-    this.partitionResponses[operationID] = new Array(requests.length);
+    this.partitionSuccessfulResponses[operationID] = new Array(requests.length);
+
+    const allowFailed = this.allowFailed;
 
     this.partitionRequests[operationID] = async.parallelLimit(
       requests.slice(0, -1).map((req, pageIndex) => async () => {
         const pageNumber = pageIndex + startingPageNumber;
         try {
           const response = await this.client!.request(req);
           if (response.status === 200) {
-            (this.partitionResponses[operationID] as Response[])[pageIndex] =
+            (this.partitionSuccessfulResponses[operationID] as Response[])[pageIndex] =
+              response.clone();
+          } else {
+            (this.partitionFailedResponses[operationID] as Response[])[pageIndex] =
               response.clone();
+              if (!allowFailed) {
+                throw new Error(`Failed to send request for page ${pageNumber}.`);
+              }
           }
         } catch (e) {
           console.error(`Failed to send request for page ${pageNumber}.`);
+          if (!allowFailed) {
+            throw e;
+          }
         }
       }),
       concurrencyLevel
@@ -189,6 +214,54 @@ export class SplitPdfHook
     return requests.at(-1) as Request;
   }
 
+    /**
+     * Forms the final response object based on the successful and failed responses.
+     * @param response - The response object returned from the API request.
+     *   Expected to be a successful response.
+     * @param successfulResponses - The list of successful responses.
+     * @param failedResponses - The list of failed responses.
+     * @returns The final response object.
+     */
+  async formFinalResponse(response: Response,
+                    successfulResponses: Response[],
+                    failedResponses: Response[]
+  ): Promise<Response>  {
+    let responseBody, responseStatus, responseStatusText;
+    const numFailedResponses = failedResponses?.length ?? 0;
+    const headers = prepareResponseHeaders(response);
+
+    if (!this.allowFailed && failedResponses && failedResponses.length > 0) {
+       const failedResponse = failedResponses[0]?.clone();
+       if (failedResponse) {
+            responseBody = await failedResponse.text();
+            responseStatus = failedResponse.status;
+            responseStatusText = failedResponse.statusText;
+        } else {
+            responseBody = JSON.stringify({"details:": "Unknown error"});
+            responseStatus = 503
+            responseStatusText = "Unknown error"
+        }
+        console.warn(
+            `${numFailedResponses} requests failed. The partition operation is cancelled.`
+          );
+      } else {
+        responseBody = await prepareResponseBody([...successfulResponses, response]);
+        responseStatus = response.status
+        responseStatusText = response.statusText
+        if (numFailedResponses > 0) {
+          console.warn(
+            `${numFailedResponses} requests failed. The results might miss some pages.`
+          );
+          }
+    }
+    return new Response(responseBody, {
+        headers: headers,
+        status: responseStatus,
+        statusText: responseStatusText,
+      });
+
+  }
+
   /**
    * Executes after a successful API request. Awaits all parallel requests and combines
    * the responses into a single response object.
@@ -203,22 +276,19 @@ export class SplitPdfHook
   ): Promise<Response> {
     const { operationID } = hookCtx;
     const responses = await this.awaitAllRequests(operationID);
-
-    if (!responses) {
+    const successfulResponses = responses?.get("success") ?? [];
+    const failedResponses = responses?.get("failed") ?? [];
+    if (!successfulResponses) {
       return response;
     }
 
-    const headers = prepareResponseHeaders(response);
-    const body = await prepareResponseBody([...responses, response]);
+    const finalResponse = await this.formFinalResponse(response, successfulResponses, failedResponses);
 
     this.clearOperation(operationID);
 
-    return new Response(body, {
-      headers: headers,
-      status: response.status,
-      statusText: response.statusText,
-    });
-  }
+    return finalResponse;
+    }
+
 
   /**
    * Executes after an unsuccessful API request. Awaits all parallel requests, if at least one
@@ -237,21 +307,19 @@ export class SplitPdfHook
   ): Promise<{ response: Response | null; error: unknown }> {
     const { operationID } = hookCtx;
     const responses = await this.awaitAllRequests(operationID);
-
-    if (!responses?.length) {
+    const successfulResponses = responses?.get("success") ?? [];
+    const failedResponses = responses?.get("failed") ?? [];
+    if (!successfulResponses?.length) {
       this.clearOperation(operationID);
       return { response, error };
     }
 
-    const okResponse = responses[0] as Response;
-    const headers = prepareResponseHeaders(okResponse);
-    const body = await prepareResponseBody(responses);
-
-    const finalResponse = new Response(body, {
-      headers: headers,
-      status: okResponse.status,
-      statusText: okResponse.statusText,
-    });
+    const okResponse = successfulResponses[0] as Response;
+    const finalResponse = await this.formFinalResponse(
+        okResponse,
+        successfulResponses.slice(1),
+        failedResponses
+    );
 
     this.clearOperation(operationID);
 
@@ -265,7 +333,8 @@ export class SplitPdfHook
    * @param operationID - The ID of the operation to clear.
    */
   clearOperation(operationID: string) {
-    delete this.partitionResponses[operationID];
+    delete this.partitionSuccessfulResponses[operationID];
+    delete this.partitionFailedResponses[operationID];
     delete this.partitionRequests[operationID];
   }
 
@@ -276,15 +345,17 @@ export class SplitPdfHook
    * @returns A promise that resolves to an array of responses, or undefined
    * if there are no requests for the given operation ID.
    */
-  async awaitAllRequests(operationID: string): Promise<Response[] | undefined> {
+  async awaitAllRequests(operationID: string): Promise<Map<string, Response[]>> {
     const requests = this.partitionRequests[operationID];
+    const responseMap = new Map<string, Response[]>();
 
     if (!requests) {
-      return;
+      return responseMap;
     }
-
     await requests;
 
-    return this.partitionResponses[operationID]?.filter((e) => e) ?? [];
+    responseMap.set("success", this.partitionSuccessfulResponses[operationID]?.filter((e) => e) ?? []);
+    responseMap.set("failed", this.partitionFailedResponses[operationID]?.filter((e) => e) ?? []);
+    return responseMap
   }
 }

src/hooks/custom/SplitPdfHook.ts

@@ -10,12 +10,14 @@ export const BASE_PROTOCOL = "https:";
 
 export const PARTITION_FORM_FILES_KEY = "files";
 export const PARTITION_FORM_SPLIT_PDF_PAGE_KEY = "split_pdf_page";
+export const PARTITION_FORM_SPLIT_PDF_ALLOW_FAILED_KEY = "split_pdf_allow_failed";
 export const PARTITION_FORM_STARTING_PAGE_NUMBER_KEY = "starting_page_number";
 export const PARTITION_FORM_SPLIT_PDF_CONCURRENCY_LEVEL =
   "split_pdf_concurrency_level";
 
 export const DEFAULT_STARTING_PAGE_NUMBER = 1;
 export const DEFAULT_NUMBER_OF_PARALLEL_REQUESTS = 8;
+export const DEFAULT_SPLIT_PDF_ALLOW_FAILED_KEY = false;
 export const MAX_NUMBER_OF_PARALLEL_REQUESTS = 15;
 
 export const MIN_PAGES_PER_THREAD = 2;

src/hooks/custom/common.ts

@@ -1,7 +1,7 @@
 import {
-  DEFAULT_NUMBER_OF_PARALLEL_REQUESTS,
+  DEFAULT_NUMBER_OF_PARALLEL_REQUESTS, DEFAULT_SPLIT_PDF_ALLOW_FAILED_KEY,
   DEFAULT_STARTING_PAGE_NUMBER,
-  MAX_NUMBER_OF_PARALLEL_REQUESTS,
+  MAX_NUMBER_OF_PARALLEL_REQUESTS, PARTITION_FORM_SPLIT_PDF_ALLOW_FAILED_KEY,
   PARTITION_FORM_SPLIT_PDF_CONCURRENCY_LEVEL,
   PARTITION_FORM_STARTING_PAGE_NUMBER_KEY,
 } from "../common.js";
@@ -41,6 +41,43 @@ function getIntegerParameter(
   return numberParameter;
 }
 
+/**
+ * Retrieves a boolean parameter from the given form data.
+ * If the parameter is not found or does not have true/false value, the default value is returned.
+ *
+ * @param formData - The form data object.
+ * @param parameterName - The name of the parameter to retrieve.
+ * @param defaultValue - The default value to use if the parameter is not found or is not
+ * a true/false string.
+ * @returns The boolean value of the parameter.
+ */
+function getBooleanParameter(
+  formData: FormData,
+  parameterName: string,
+  defaultValue: boolean
+): boolean {
+  let booleanParameter = defaultValue;
+  const formDataParameter = formData.get(parameterName);
+
+  if (formDataParameter === null) {
+    return booleanParameter;
+  }
+
+  const formDataBooleanParameterString = formDataParameter as string;
+
+  if (formDataBooleanParameterString.toLowerCase() === "true") {
+    booleanParameter = true;
+  } else if (formDataBooleanParameterString.toLowerCase() === "false") {
+    booleanParameter = false;
+  } else {
+    console.warn(
+      `'${parameterName}' is not a valid boolean. Using default value '${defaultValue}'.`
+    );
+  }
+
+  return booleanParameter;
+}
+
 /**
  * Gets the number of maximum requests that can be made when splitting PDF.
  * - The number of maximum requests is determined by the value of the request parameter
@@ -82,6 +119,48 @@ export function getSplitPdfConcurrencyLevel(formData: FormData): number {
   return splitPdfConcurrencyLevel;
 }
 
+/**
+ * Gets the allowFailed parameter which decides whether the partial requests can fail or not
+ * when using splitPdfPage parameter.
+ * - The number of maximum requests is determined by the value of the request parameter
+ * `split_pdf_thread`.
+ * - If the parameter is not set or has an invalid value, the default number of
+ * parallel requests (5) is used.
+ * - If the number of maximum requests is greater than the maximum allowed (15), it is
+ * clipped to the maximum value.
+ * - If the number of maximum requests is less than 1, the default number of parallel
+ * requests (5) is used.
+ *
+ * @returns The number of maximum requests to use when calling the API to split a PDF.
+ */
+export function getSplitPdfAllowFailed(formData: FormData): boolean {
+  const splitPdfAllowFailed = getBooleanParameter(
+    formData,
+    PARTITION_FORM_SPLIT_PDF_ALLOW_FAILED_KEY,
+    DEFAULT_SPLIT_PDF_ALLOW_FAILED_KEY
+  );
+
+
+  if (splitPdfAllowFailed) {
+    console.info(
+    `Running split PDF requests in parallel with no-strict mode - 
+      the failed requests will not stop the process, and the resulting elements might miss
+      some pages in case of failure.`
+    );
+  } else {
+    console.info(
+    `Running split PDF requests in parallel with strict mode - 
+      the failed requests will stop the process, and the resulting elements will have all pages
+      or error out.`
+    )
+  }
+
+  console.info(
+    `Set ${PARTITION_FORM_SPLIT_PDF_CONCURRENCY_LEVEL} parameter if you want to change that.`
+  );
+  return splitPdfAllowFailed;
+}
+
 /**
  * Retrieves the starting page number from the provided form data.
  * If the starting page number is not a valid integer or less than 1,

src/hooks/custom/utils/form.ts

@@ -136,6 +136,10 @@ export type PartitionParameters = {
      * Number of maximum concurrent requests made when splitting PDF. Ignored on backend.
      */
     splitPdfConcurrencyLevel?: number | undefined;
+    /**
+     * Controls the split pdf strict mode - if set to True, the partial requests with page batches may fail. If set to False - the first failed partial request will cancel the process and error out.
+     */
+    splitPdfAllowFailed?: boolean | undefined;
     /**
      * Should the pdf file be split at client. Ignored on backend.
      */