Improve performance for parsing wf input (#889)

 improve performance for parsing wf input

Author: Assem-Uber

src/utils/data-formatters/__tests__/format-input-payload.test.ts

@@ -32,6 +32,11 @@ describe('formatInputPayload', () => {
     expect(formatInputPayload(input)).toEqual(expected);
   });
   // end of empty data checks
+  test('should handle base64 encoded JSON with boolean value', () => {
+    const input = { data: btoa(`true`) };
+    const expected = [true];
+    expect(formatInputPayload(input)).toEqual(expected);
+  });
 
   test('should parse base64 encoded JSON lines separated by \n correctly', () => {
     const input = {
@@ -106,17 +111,44 @@ describe('formatInputPayload', () => {
     expect(formatInputPayload(input)).toEqual(expected);
   });
 
+  test('should handle base64 encoded JSON with exponential numbers', () => {
+    const input = {
+      data: btoa(`[333e-10]\n{"number": 1.23e-10} -1.23e+10\n1e20`),
+    };
+    const expected = [[333e-10], { number: 1.23e-10 }, -1.23e10, 1e20];
+    expect(formatInputPayload(input)).toEqual(expected);
+  });
+
   test('should handle base64 encoded multiple mixed JSON objects', () => {
     const input = {
       data: btoa(
-        `{"a": 1, "b": "text"}\n{"c": [1, 2, 3], "d": null}\n42\n"Hello"`
+        `{"a": 1, "b": "text"} 0\n{"c": [1, 2, 3], "d": null}\n42\n"Hello"\ntrue`
       ),
     };
     const expected = [
       { a: 1, b: 'text' },
+      0,
       { c: [1, 2, 3], d: null },
       42,
       'Hello',
+      true,
+    ];
+    expect(formatInputPayload(input)).toEqual(expected);
+  });
+
+  test('should handle base64 encoded JSON with nested json string', () => {
+    const stringifiedJson = JSON.stringify(
+      `{ street: ['Hello', 1, { a: 'b' }] }`
+    );
+    const input = {
+      data: btoa(`{"name": "John", "age": 30, "address": ${stringifiedJson}}`),
+    };
+    const expected = [
+      {
+        name: 'John',
+        age: 30,
+        address: `{ street: ['Hello', 1, { a: 'b' }] }`,
+      },
     ];
     expect(formatInputPayload(input)).toEqual(expected);
   });

src/utils/data-formatters/format-input-payload.ts

@@ -1,62 +1,95 @@
 import logger from '@/utils/logger';
 
 import losslessJsonParse from '../lossless-json-parse';
+
+const separators = ['\n', ' '];
+
 const formatInputPayload = (
   payload: { data?: string | null } | null | undefined
 ) => {
   const data = payload?.data;
-
   if (!data) {
     return null;
   }
 
   const parsedData = atob(data);
-  return parseJsonLines(parsedData);
+  return parseMultipleInputs(parsedData);
 };
 
-function parseJsonLines(input: string) {
-  const jsonArray = [];
-  let currentJson = '';
-  const separators = ['\n', ' '];
-
-  for (let i = 0; i < input.length; i++) {
-    const char = input[i];
-    if (separators.includes(char)) {
-      // Try to parse the current JSON string
-      if (currentJson) {
-        try {
-          const jsonObject = losslessJsonParse(currentJson);
-          // If successful, add the object to the array
-          jsonArray.push(jsonObject);
-          // Reset currentJson for the next JSON object
-          currentJson = '';
-        } catch {
-          // If parsing fails, treat the separator as part of the currentJson and continue with the next char
-          currentJson += char;
-        }
-      }
-    } else {
-      currentJson += char;
+const parseMultipleInputs = (input: string) => {
+  const results = [];
+  let currentIndex = 0;
+
+  while (currentIndex < input.length) {
+    while (separators.includes(input[currentIndex])) {
+      currentIndex++;
     }
-  }
+    if (currentIndex >= input.length) break;
 
-  // Handle case where the last JSON object might be malformed
-  if (currentJson.trim() !== '') {
     try {
-      const jsonObject = losslessJsonParse(currentJson);
-      jsonArray.push(jsonObject);
-    } catch {
+      const { startIndex, endIndex, jsonString } = extractJsonValue(
+        input,
+        currentIndex
+      );
+
+      if (endIndex > startIndex) {
+        // move to the end of the string before parsing to avoid parsing the same string if the parsing fails
+        currentIndex = endIndex;
+        results.push(losslessJsonParse(jsonString));
+      } else {
+        currentIndex++;
+      }
+    } catch (error) {
       logger.error(
         {
           input,
-          currentJson,
+          currentIndex,
+          error,
         },
         'Error parsing JSON string'
       );
     }
   }
 
-  return jsonArray;
-}
+  return results;
+};
+
+const extractJsonValue = (input: string, startIndex: number) => {
+  let endIndex = startIndex;
+  let openBrackets = 0;
+  let openBraces = 0;
+  let inString = false;
+  let escapeNext = false; // used to handle escaped quotes
+
+  while (endIndex < input.length) {
+    const char = input[endIndex];
 
+    if (escapeNext) {
+      escapeNext = false;
+    } else if (char === '\\') {
+      escapeNext = true;
+    } else if (char === '"' && !inString) {
+      inString = true;
+    } else if (char === '"' && inString) {
+      inString = false;
+    } else if (!inString) {
+      if (char === '[') openBrackets++;
+      if (char === ']') openBrackets--;
+      if (char === '{') openBraces++;
+      if (char === '}') openBraces--;
+
+      // When a separator is found, this indicates the end of a JSON value if we are not inside any array or object
+      if (separators.includes(char) && openBrackets === 0 && openBraces === 0) {
+        break;
+      }
+    }
+    endIndex++;
+  }
+
+  return {
+    startIndex,
+    endIndex,
+    jsonString: input.slice(startIndex, endIndex),
+  };
+};
 export default formatInputPayload;