Skip to content

perf: [10196] improve perfromance of CSV parsing table #325

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

perf: [10196] improve perfromance of CSV parsing table #325

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a2d6101
perf: [10196] build a dict of the colums metadata before the main par…
arianber7 Oct 15, 2025
630ca95
perf: [10196] add custom function to parse dates
arianber7 Oct 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 53 additions & 24 deletions src/FileUtils/AgGridUtils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,39 +85,68 @@ const _getColTypeFromTypeArray = (typeArray) => {
return 'string'; // Fall back to default type
};

const validateDateFormat = (value, options = {}) => {
if (typeof value !== 'string') return false;
const { dateFormat = 'yyyy-MM-dd' } = options;

const formatRegex = dateFormat
.replace(/yyyy/, '\\d{4}')
.replace(/MM/, '(0[1-9]|1[0-2])')
.replace(/dd/, '(0[1-9]|[12]\\d|3[01])')
.replace(/\//g, '\\/')
.replace(/-/g, '-');

const regex = new RegExp(`^${formatRegex}$`);
return regex.test(value);
};

const _validateFormat = (rows, hasHeader, cols, options) => {
const colsData = cols.map((col) => ({ ...col, type: _getColTypeFromTypeArray(col.type) }));
const errors = [];
const knownColsCount = colsData.length;
const startIndex = hasHeader ? 1 : 0;

const colMeta = cols.map((col) => ({
field: col.field,
type: _getColTypeFromTypeArray(col.type),
acceptsEmptyFields: col.acceptsEmptyFields ?? col.cellEditorParams?.acceptsEmptyFields ?? false,
colOptions: {
...options,
enumValues: col.enumValues ?? col.cellEditorParams?.enumValues,
minValue: col.minValue,
maxValue: col.maxValue,
},
}));

for (let rowIndex = startIndex; rowIndex < rows.length; rowIndex++) {
const row = rows[rowIndex];
while (row[row.length - 1] === undefined && row.length > knownColsCount) row.pop();
if (row.length !== knownColsCount || row.includes(undefined))
_forgeColumnsCountError(row, rowIndex + 1, colsData, errors);
row.forEach((rowCell, colIndex) => {
if (colIndex < knownColsCount) {
const colType = colsData[colIndex].type;
if (colType && rowCell !== undefined) {
// use of cellEditorParams is deprecated
const colOptions = {
...options,
enumValues: colsData[colIndex]?.enumValues ?? colsData[colIndex]?.cellEditorParams?.enumValues,
minValue: colsData[colIndex]?.minValue,
maxValue: colsData[colIndex]?.maxValue,
};
const acceptsEmptyFields =
// use of cellEditorParams is deprecated
colsData[colIndex].acceptsEmptyFields ?? colsData[colIndex].cellEditorParams?.acceptsEmptyFields ?? false;
const validationResult = ValidationUtils.isValid(rowCell, colType, colOptions, acceptsEmptyFields);
if (validationResult !== true) {
const { summary: errorSummary, context: errorContext } = validationResult;
const errorLoc = `Line ${rowIndex + 1}, Column ${colIndex + 1} ("${colsData[colIndex].field}")`;
errors.push(new PanelError(errorSummary, errorLoc, errorContext));
}
}
if (row.length !== knownColsCount || row.includes(undefined)) {
_forgeColumnsCountError(row, rowIndex + 1, cols, errors);
continue;
}

for (let colIndex = 0; colIndex < knownColsCount; colIndex++) {
const { type, colOptions, acceptsEmptyFields, field } = colMeta[colIndex];
const value = row[colIndex];
if (value === undefined) continue;

if (type === 'date' && !validateDateFormat(value, colOptions)) {
const expectedFormat = colOptions.dateFormat || 'dd/MM/yyyy';
const errorSummary = 'Incorrect date value';
const errorContext = `Incorrect value: "${value}" for type date\nExpected format: ${expectedFormat}`;
const errorLoc = `Line ${rowIndex + 1}, Column ${colIndex + 1} ("${field}")`;
errors.push(new PanelError(errorSummary, errorLoc, errorContext));
continue;
}

const validationResult = ValidationUtils.isValid(value, type, colOptions, acceptsEmptyFields);
if (validationResult !== true) {
const { summary: errorSummary, context: errorContext } = validationResult;
const errorLoc = `Line ${rowIndex + 1}, Column ${colIndex + 1} ("${field}")`;
errors.push(new PanelError(errorSummary, errorLoc, errorContext));
}
});
}
}

return errors;
Expand Down