merge: merge branch 'abe/Improve-perfs-of-CSV-parsing-for-Table-components-PROD-10196'

Author: csm-thu

src/FileUtils/AgGridUtils.js

@@ -1,5 +1,6 @@
 // Copyright (c) Cosmo Tech.
 // Licensed under the MIT license.
+import DateUtils from '../DateUtils/DateUtils';
 import { ValidationUtils } from '../ValidationUtils';
 import { Error as PanelError } from '../models';
 import CSV from './CSVUtils';
@@ -86,38 +87,50 @@ const _getColTypeFromTypeArray = (typeArray) => {
 };
 
 const _validateFormat = (rows, hasHeader, cols, options) => {
-  const colsData = cols.map((col) => ({ ...col, type: _getColTypeFromTypeArray(col.type) }));
   const errors = [];
-  const knownColsCount = colsData.length;
+  const knownColsCount = cols.length;
   const startIndex = hasHeader ? 1 : 0;
+
+  const colMeta = cols.map((col) => ({
+    field: col.field,
+    type: _getColTypeFromTypeArray(col.type),
+    acceptsEmptyFields: col.acceptsEmptyFields ?? col.cellEditorParams?.acceptsEmptyFields ?? false,
+    colOptions: {
+      ...options,
+      enumValues: col.enumValues ?? col.cellEditorParams?.enumValues,
+      minValue: col.minValue,
+      maxValue: col.maxValue,
+    },
+  }));
+  // For date columns, convert min & max values to do it only once
+  colMeta.forEach((col) => {
+    if (col.type === 'date' && options?.dateFormat) {
+      const colOptions = col.colOptions;
+      if (colOptions.minValue != null) colOptions.minDate = DateUtils.parse(colOptions.minValue, options?.dateFormat);
+      if (colOptions.maxValue != null) colOptions.maxDate = DateUtils.parse(colOptions.maxValue, options?.dateFormat);
+    }
+  });
+
   for (let rowIndex = startIndex; rowIndex < rows.length; rowIndex++) {
     const row = rows[rowIndex];
     while (row[row.length - 1] === undefined && row.length > knownColsCount) row.pop();
-    if (row.length !== knownColsCount || row.includes(undefined))
-      _forgeColumnsCountError(row, rowIndex + 1, colsData, errors);
-    row.forEach((rowCell, colIndex) => {
-      if (colIndex < knownColsCount) {
-        const colType = colsData[colIndex].type;
-        if (colType && rowCell !== undefined) {
-          // use of cellEditorParams is deprecated
-          const colOptions = {
-            ...options,
-            enumValues: colsData[colIndex]?.enumValues ?? colsData[colIndex]?.cellEditorParams?.enumValues,
-            minValue: colsData[colIndex]?.minValue,
-            maxValue: colsData[colIndex]?.maxValue,
-          };
-          const acceptsEmptyFields =
-            // use of cellEditorParams is deprecated
-            colsData[colIndex].acceptsEmptyFields ?? colsData[colIndex].cellEditorParams?.acceptsEmptyFields ?? false;
-          const validationResult = ValidationUtils.isValid(rowCell, colType, colOptions, acceptsEmptyFields);
-          if (validationResult !== true) {
-            const { summary: errorSummary, context: errorContext } = validationResult;
-            const errorLoc = `Line ${rowIndex + 1}, Column ${colIndex + 1} ("${colsData[colIndex].field}")`;
-            errors.push(new PanelError(errorSummary, errorLoc, errorContext));
-          }
-        }
+
+    if (row.length !== knownColsCount || row.includes(undefined)) {
+      _forgeColumnsCountError(row, rowIndex + 1, cols, errors);
+    }
+
+    for (let colIndex = 0; colIndex < knownColsCount; colIndex++) {
+      const { type, colOptions, acceptsEmptyFields, field } = colMeta[colIndex];
+      const value = row[colIndex];
+      if (value === undefined) continue;
+
+      const validationResult = ValidationUtils.isValid(value, type, colOptions, acceptsEmptyFields);
+      if (validationResult !== true) {
+        const { summary: errorSummary, context: errorContext } = validationResult;
+        const errorLoc = `Line ${rowIndex + 1}, Column ${colIndex + 1} ("${field}")`;
+        errors.push(new PanelError(errorSummary, errorLoc, errorContext));
       }
-    });
+    }
   }
 
   return errors;

src/FileUtils/__test__/AgGridUtils.spec.js

@@ -83,6 +83,30 @@ describe('parse valid CSV strings', () => {
   });
 });
 
+describe('regression: invalid number values should trigger validation errors', () => {
+  const options = { dateFormat: 'dd/MM/yyyy' };
+
+  test('should raise error for bad integer value in age column', () => {
+    const headerCols = AgGridUtils.getFlattenColumnsWithoutGroups(CUSTOMERS_COLS).map((col) => col.field);
+
+    const invalidRow = headerCols.map((field) => {
+      if (field === 'age') return 'bad_int';
+      if (field === 'birthday') return '03/05/1978';
+      if (field === 'height') return '1.7';
+      return '';
+    });
+
+    const csvData = [headerCols, invalidRow];
+    const csvStr = csvData.map((row) => row.join(',')).join('\n');
+
+    const result = AgGridUtils.fromCSV(csvStr, true, CUSTOMERS_COLS, options);
+
+    expect(result.error).toBeDefined();
+    expect(result.error.length).toBeGreaterThan(0);
+    expect(result.error[0].summary.toLowerCase()).toMatch(/(type|incorrect|int|empty)/);
+  });
+});
+
 describe('parse with invalid parameters', () => {
   test('missing fields definition', () => {
     const res = AgGridUtils.fromCSV('', false, undefined);

src/FileUtils/__test__/CustomersData.js

@@ -73,7 +73,7 @@ export const CUSTOMERS_COLS = [
   {
     headerName: 'identity',
     children: [
-      { field: 'birthday', type: ['date'], minValue: new Date('1900-01-01'), maxValue: new Date('2030-01-01') },
+      { field: 'birthday', type: ['date'], minValue: '01/01/1900', maxValue: '01/01/2030' },
       { field: 'height', type: ['number'], minValue: 0, maxValue: 2.5 },
     ],
   },
@@ -88,7 +88,7 @@ export const CUSTOMERS_COLS_DEPRECATED = [
     type: ['enum'],
     cellEditorParams: { enumValues: ['AppleJuice', 'Beer', 'OrangeJuice', 'Wine'] },
   },
-  { field: 'birthday', type: ['date'], minValue: new Date('1900-01-01'), maxValue: new Date('2030-01-01') },
+  { field: 'birthday', type: ['date'], minValue: '01/01/1900', maxValue: '01/01/2030' },
   { field: 'height', type: ['number'], minValue: 0, maxValue: 2.5 },
 ];
 

src/ValidationUtils/ValidationUtils.js

@@ -69,9 +69,7 @@ const castToDate = (dateOrStrValue, dateFormat) => {
   return DateUtils.parse(dateOrStrValue, dateFormat);
 };
 
-const isDateInRange = (value, minValue, maxValue, dateFormat) => {
-  const minDate = castToDate(minValue, dateFormat);
-  const maxDate = castToDate(maxValue, dateFormat);
+const isDateInRange = (value, minDate, maxDate, dateFormat) => {
   const format = DateUtils.format;
   if (value == null) return null;
   if (dateFormat == null) return forgeConfigError("Missing option dateFormat, can't perform date validation.");
@@ -95,9 +93,13 @@ const isValid = (dataStr, type, options, canBeEmpty = false) => {
       return isBool(dataStr) || forgeTypeError(dataStr, type, options);
     case 'date': {
       if (!options?.dateFormat) return forgeConfigError("Missing option dateFormat, can't perform date validation.");
-      if (!isDate(dataStr, options?.dateFormat)) return forgeTypeError(dataStr, type, options);
+
       const valueAsDate = DateUtils.parse(dataStr, options?.dateFormat);
-      return isDateInRange(valueAsDate, options?.minValue, options?.maxValue, options?.dateFormat);
+      if (isNaN(valueAsDate.getTime())) return forgeTypeError(dataStr, type, options); // Invalid date
+
+      const minDate = options?.minDate ?? castToDate(options?.minValue, options?.dateFormat);
+      const maxDate = options?.maxDate ?? castToDate(options?.maxValue, options?.dateFormat);
+      return isDateInRange(valueAsDate, minDate, maxDate, options?.dateFormat);
     }
     case 'enum':
       if (!options.enumValues) return forgeConfigError("Missing option enumValues, can't perform enum validation.");