Skip to content

Thu/range validation on table upload #323

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 22, 2025
Merged

Thu/range validation on table upload #323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dc51f6f
test: set deterministic maxValue dates in test fixtures
csm-thu Jan 20, 2025
ce4ba32
fix!: modify isValid function to return error structures on validatio…
csm-thu Jan 21, 2025
c7f0b3d
feat: add min & max validation in ValidationUtils and AgGridUtils fil…
csm-thu Jan 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 11 additions & 24 deletions src/FileUtils/AgGridUtils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,16 +54,6 @@ const _buildNumberColumnsError = (rowLineNumber, expectedCols, row) => {
return new PanelError(errorSummary, errorLoc, errorContext);
};

const _buildTypeError = (type, rowLineNumber, colIndex, colsData, value, expected) => {
const errorSummary = `Incorrect ${type} value`;
const errorLoc = `Line ${rowLineNumber}, Column ${colIndex + 1} ("${colsData[colIndex].field}")`;
const errorContext = `${errorSummary} (${errorLoc})\n` + `Incorrect value : "${value}" for type ${type}`;
if (!expected || expected.length === 0) {
return new PanelError(errorSummary, errorLoc, errorContext);
}
return new PanelError(errorSummary, errorLoc, errorContext + '\n' + expected);
};

const DEFAULT_CSV_EXPORT_OPTIONS = {
colSep: ',',
dateFormat: 'yyyy-MM-dd',
Expand All @@ -82,16 +72,6 @@ const _forgeColumnsCountError = (row, rowLineNumber, expectedCols, errors) => {
if (row.length !== expectedCols.length) errors.push(_buildNumberColumnsError(rowLineNumber, expectedCols, row));
};

const _forgeTypeError = (value, rowLineNumber, type, options, colsData, colIndex) => {
let expected = '';
if (type === 'enum') {
expected = `Expected values: [${options.enumValues.join()}]`;
} else if (type === 'date') {
expected = `Expected format: ${options.dateFormat}`;
}
return _buildTypeError(type, rowLineNumber, colIndex, colsData, value, expected);
};

const _getColTypeFromTypeArray = (typeArray) => {
if (!typeArray || typeArray.length === 0) {
return 'string'; // Fall back to default type
Expand Down Expand Up @@ -120,13 +100,20 @@ const _validateFormat = (rows, hasHeader, cols, options) => {
const colType = colsData[colIndex].type;
if (colType && rowCell !== undefined) {
// use of cellEditorParams is deprecated
const enumValues = colsData[colIndex]?.enumValues ?? colsData[colIndex]?.cellEditorParams?.enumValues;
const colOptions = { ...options, enumValues };
const colOptions = {
...options,
enumValues: colsData[colIndex]?.enumValues ?? colsData[colIndex]?.cellEditorParams?.enumValues,
minValue: colsData[colIndex]?.minValue,
maxValue: colsData[colIndex]?.maxValue,
};
const acceptsEmptyFields =
// use of cellEditorParams is deprecated
colsData[colIndex].acceptsEmptyFields ?? colsData[colIndex].cellEditorParams?.acceptsEmptyFields ?? false;
if (!ValidationUtils.isValid(rowCell, colType, colOptions, acceptsEmptyFields)) {
errors.push(_forgeTypeError(rowCell, rowIndex + 1, colType, colOptions, colsData, colIndex));
const validationResult = ValidationUtils.isValid(rowCell, colType, colOptions, acceptsEmptyFields);
if (validationResult !== true) {
const { summary: errorSummary, context: errorContext } = validationResult;
const errorLoc = `Line ${rowIndex + 1}, Column ${colIndex + 1} ("${colsData[colIndex].field}")`;
errors.push(new PanelError(errorSummary, errorLoc, errorContext));
}
}
}
Expand Down
112 changes: 46 additions & 66 deletions src/FileUtils/__test__/CustomersData.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ export const CUSTOMERS_COLS = [
{
headerName: 'identity',
children: [
{ field: 'birthday', type: ['date'], minValue: '1900-01-01', maxValue: new Date().toISOString() },
{ field: 'birthday', type: ['date'], minValue: new Date('1900-01-01'), maxValue: new Date('2030-01-01') },
{ field: 'height', type: ['number'], minValue: 0, maxValue: 2.5 },
],
},
Expand All @@ -88,7 +88,7 @@ export const CUSTOMERS_COLS_DEPRECATED = [
type: ['enum'],
cellEditorParams: { enumValues: ['AppleJuice', 'Beer', 'OrangeJuice', 'Wine'] },
},
{ field: 'birthday', type: ['date'], minValue: '1900-01-01', maxValue: new Date().toISOString() },
{ field: 'birthday', type: ['date'], minValue: new Date('1900-01-01'), maxValue: new Date('2030-01-01') },
{ field: 'height', type: ['number'], minValue: 0, maxValue: 2.5 },
];

Expand Down Expand Up @@ -210,6 +210,8 @@ export const INVALID_CUSTOMERS_ROWS = [
['Bob', '15', '1', 'AppleJuice', 'bad_date', '1.70'],
['Bob', '15', '0', 'AppleJuice', '01/01/2000', 'bad_number'],
['Bob', 'bad_int', 'bad_bool', 'bad_enum', 'bad_date', 'bad_number'],
['Bob', '-1', 'yes', 'AppleJuice', '01/01/1899', '-0.1'], // minValue not respected
['Bob', '999', 'yes', 'AppleJuice', '01/01/2999', '99.9'], // maxValue not respected
['Bob', '15', 'yes', 'AppleJuice', '01/01/2000', '1.70'],
];

Expand All @@ -221,64 +223,55 @@ export const EXPECTED_ERRORS_WITH_HEADER = [
'Expected data format : "name,age,canDrinkAlcohol,favoriteDrink,birthday,height"\n' +
'Incorrect Row : "MissingColumns"'
),
new Error(
'Incorrect int value',
'Line 3, Column 2 ("age")',
'Incorrect int value (Line 3, Column 2 ("age"))\nIncorrect value : "bad_int" for type int'
),
new Error('Incorrect int value', 'Line 3, Column 2 ("age")', 'Incorrect value: "bad_int" for type int'),
new Error(
'Incorrect bool value',
'Line 4, Column 3 ("canDrinkAlcohol")',
'Incorrect bool value (Line 4, Column 3 ("canDrinkAlcohol"))\nIncorrect value : "bad_bool" for type bool'
'Incorrect value: "bad_bool" for type bool'
),
new Error(
'Incorrect enum value',
'Line 5, Column 4 ("favoriteDrink")',
'Incorrect enum value (Line 5, Column 4 ("favoriteDrink"))\n' +
'Incorrect value : "bad_enum" for type enum\n' +
'Expected values: [AppleJuice,Beer,OrangeJuice,Wine]'
'Incorrect value: "bad_enum" for type enum\n' + 'Expected values: [AppleJuice,Beer,OrangeJuice,Wine]'
),
new Error(
'Incorrect date value',
'Line 6, Column 5 ("birthday")',
'Incorrect date value (Line 6, Column 5 ("birthday"))\n' +
'Incorrect value : "bad_date" for type date\n' +
'Expected format: dd/MM/yyyy'
),
new Error(
'Incorrect number value',
'Line 7, Column 6 ("height")',
'Incorrect number value (Line 7, Column 6 ("height"))\nIncorrect value : "bad_number" for type number'
),
new Error(
'Incorrect int value',
'Line 8, Column 2 ("age")',
'Incorrect int value (Line 8, Column 2 ("age"))\nIncorrect value : "bad_int" for type int'
'Incorrect value: "bad_date" for type date\n' + 'Expected format: dd/MM/yyyy'
),
new Error('Incorrect number value', 'Line 7, Column 6 ("height")', 'Incorrect value: "bad_number" for type number'),
new Error('Incorrect int value', 'Line 8, Column 2 ("age")', 'Incorrect value: "bad_int" for type int'),
new Error(
'Incorrect bool value',
'Line 8, Column 3 ("canDrinkAlcohol")',
'Incorrect bool value (Line 8, Column 3 ("canDrinkAlcohol"))\nIncorrect value : "bad_bool" for type bool'
'Incorrect value: "bad_bool" for type bool'
),
new Error(
'Incorrect enum value',
'Line 8, Column 4 ("favoriteDrink")',
'Incorrect enum value (Line 8, Column 4 ("favoriteDrink"))\n' +
'Incorrect value : "bad_enum" for type enum\n' +
'Expected values: [AppleJuice,Beer,OrangeJuice,Wine]'
'Incorrect value: "bad_enum" for type enum\n' + 'Expected values: [AppleJuice,Beer,OrangeJuice,Wine]'
),
new Error(
'Incorrect date value',
'Line 8, Column 5 ("birthday")',
'Incorrect date value (Line 8, Column 5 ("birthday"))\n' +
'Incorrect value : "bad_date" for type date\n' +
'Expected format: dd/MM/yyyy'
'Incorrect value: "bad_date" for type date\n' + 'Expected format: dd/MM/yyyy'
),
new Error('Incorrect number value', 'Line 8, Column 6 ("height")', 'Incorrect value: "bad_number" for type number'),
// Min/max values errors
new Error('Value out of range', 'Line 9, Column 2 ("age")', 'Value "-1" should be greater than 0'),
new Error(
'Value out of range',
'Line 9, Column 5 ("birthday")',
'Value "01/01/1899" should be greater than 01/01/1900'
),
new Error('Value out of range', 'Line 9, Column 6 ("height")', 'Value "-0.1" should be greater than 0'),
new Error('Value out of range', 'Line 10, Column 2 ("age")', 'Value "999" should be less than 120'),
new Error(
'Incorrect number value',
'Line 8, Column 6 ("height")',
'Incorrect number value (Line 8, Column 6 ("height"))\nIncorrect value : "bad_number" for type number'
'Value out of range',
'Line 10, Column 5 ("birthday")',
'Value "01/01/2999" should be less than 01/01/2030'
),
new Error('Value out of range', 'Line 10, Column 6 ("height")', 'Value "99.9" should be less than 2.5'),
];

export const EXPECTED_ERRORS_WITHOUT_COLS = [
Expand All @@ -298,64 +291,51 @@ export const EXPECTED_ERRORS_WITHOUT_HEADER = [
'Expected data format : "name,age,canDrinkAlcohol,favoriteDrink,birthday,height"\n' +
'Incorrect Row : "MissingColumns"'
),
new Error(
'Incorrect int value',
'Line 2, Column 2 ("age")',
'Incorrect int value (Line 2, Column 2 ("age"))\nIncorrect value : "bad_int" for type int'
),
new Error('Incorrect int value', 'Line 2, Column 2 ("age")', 'Incorrect value: "bad_int" for type int'),
new Error(
'Incorrect bool value',
'Line 3, Column 3 ("canDrinkAlcohol")',
'Incorrect bool value (Line 3, Column 3 ("canDrinkAlcohol"))\nIncorrect value : "bad_bool" for type bool'
'Incorrect value: "bad_bool" for type bool'
),
new Error(
'Incorrect enum value',
'Line 4, Column 4 ("favoriteDrink")',
'Incorrect enum value (Line 4, Column 4 ("favoriteDrink"))\n' +
'Incorrect value : "bad_enum" for type enum\n' +
'Expected values: [AppleJuice,Beer,OrangeJuice,Wine]'
'Incorrect value: "bad_enum" for type enum\n' + 'Expected values: [AppleJuice,Beer,OrangeJuice,Wine]'
),
new Error(
'Incorrect date value',
'Line 5, Column 5 ("birthday")',
'Incorrect date value (Line 5, Column 5 ("birthday"))\n' +
'Incorrect value : "bad_date" for type date\n' +
'Expected format: dd/MM/yyyy'
),
new Error(
'Incorrect number value',
'Line 6, Column 6 ("height")',
'Incorrect number value (Line 6, Column 6 ("height"))\nIncorrect value : "bad_number" for type number'
),
new Error(
'Incorrect int value',
'Line 7, Column 2 ("age")',
'Incorrect int value (Line 7, Column 2 ("age"))\nIncorrect value : "bad_int" for type int'
'Incorrect value: "bad_date" for type date\n' + 'Expected format: dd/MM/yyyy'
),
new Error('Incorrect number value', 'Line 6, Column 6 ("height")', 'Incorrect value: "bad_number" for type number'),
new Error('Incorrect int value', 'Line 7, Column 2 ("age")', 'Incorrect value: "bad_int" for type int'),
new Error(
'Incorrect bool value',
'Line 7, Column 3 ("canDrinkAlcohol")',
'Incorrect bool value (Line 7, Column 3 ("canDrinkAlcohol"))\nIncorrect value : "bad_bool" for type bool'
'Incorrect value: "bad_bool" for type bool'
),
new Error(
'Incorrect enum value',
'Line 7, Column 4 ("favoriteDrink")',
'Incorrect enum value (Line 7, Column 4 ("favoriteDrink"))\n' +
'Incorrect value : "bad_enum" for type enum\n' +
'Expected values: [AppleJuice,Beer,OrangeJuice,Wine]'
'Incorrect value: "bad_enum" for type enum\n' + 'Expected values: [AppleJuice,Beer,OrangeJuice,Wine]'
),
new Error(
'Incorrect date value',
'Line 7, Column 5 ("birthday")',
'Incorrect date value (Line 7, Column 5 ("birthday"))\n' +
'Incorrect value : "bad_date" for type date\n' +
'Expected format: dd/MM/yyyy'
'Incorrect value: "bad_date" for type date\n' + 'Expected format: dd/MM/yyyy'
),
new Error('Incorrect number value', 'Line 7, Column 6 ("height")', 'Incorrect value: "bad_number" for type number'),
// Min/max values errors
new Error('Value out of range', 'Line 8, Column 2 ("age")', 'Value "-1" should be greater than 0'),
new Error(
'Incorrect number value',
'Line 7, Column 6 ("height")',
'Incorrect number value (Line 7, Column 6 ("height"))\nIncorrect value : "bad_number" for type number'
'Value out of range',
'Line 8, Column 5 ("birthday")',
'Value "01/01/1899" should be greater than 01/01/1900'
),
new Error('Value out of range', 'Line 8, Column 6 ("height")', 'Value "-0.1" should be greater than 0'),
new Error('Value out of range', 'Line 9, Column 2 ("age")', 'Value "999" should be less than 120'),
new Error('Value out of range', 'Line 9, Column 5 ("birthday")', 'Value "01/01/2999" should be less than 01/01/2030'),
new Error('Value out of range', 'Line 9, Column 6 ("height")', 'Value "99.9" should be less than 2.5'),
];

export const EXPECTED_CUSTOM_CSV_OUTPUT =
Expand Down
91 changes: 74 additions & 17 deletions src/ValidationUtils/ValidationUtils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,22 @@
// Licensed under the MIT license.
import validator from 'validator';
import { DateUtils } from '../DateUtils';
import { Error as PanelError } from '../models';

const forgeTypeError = (value, type, options) => {
let expected;
if (type === 'enum') expected = `Expected values: [${options.enumValues.join()}]`;
else if (type === 'date') expected = `Expected format: ${options?.dateFormat}`;

const error = new PanelError(`Incorrect ${type} value`, null, `Incorrect value: "${value}" for type ${type}`);
if (expected) error.context += '\n' + expected;
return error;
};

const forgeConfigError = (errorContext) => {
console.warn(`Configuration error: ${errorContext}`);
return { summary: 'Configuration error', context: errorContext };
};

const isBool = (dataStr) => {
return validator.isBoolean(dataStr, { loose: true });
Expand All @@ -27,34 +43,75 @@ const isString = (data) => {
return typeof data === 'string';
};

const isInRange = (value, minValue, maxValue) => {
if (value == null) return null;

let errorMessage;
if (minValue != null && value < minValue) errorMessage = `Value "${value}" should be greater than ${minValue}`;
if (maxValue != null && value > maxValue) errorMessage = `Value "${value}" should be less than ${maxValue}`;

if (errorMessage == null) return true;
return new PanelError(`Value out of range`, null, errorMessage);
};

const castToDate = (dateOrStrValue, dateFormat) => {
if (dateOrStrValue == null) return;
if (dateOrStrValue instanceof Date) return dateOrStrValue;
if (typeof dateOrStrValue !== 'string') {
console.warn(`Configuration error: ${dateOrStrValue} is not a string nor a Date.`);
return;
}
if (!isDate(dateOrStrValue, dateFormat)) {
console.warn(`Configuration error: ${forgeTypeError(dateOrStrValue, 'date', { dateFormat }).context}.`);
return;
}

return DateUtils.parse(dateOrStrValue, dateFormat);
};

const isDateInRange = (value, minValue, maxValue, dateFormat) => {
const minDate = castToDate(minValue, dateFormat);
const maxDate = castToDate(maxValue, dateFormat);
const format = DateUtils.format;
if (value == null) return null;
if (dateFormat == null) return forgeConfigError("Missing option dateFormat, can't perform date validation.");

let errorMessage;
if (minDate != null && value < minDate)
errorMessage = `Value "${format(value, dateFormat)}" should be greater than ${format(minDate, dateFormat)}`;
if (maxDate != null && value > maxDate)
errorMessage = `Value "${format(value, dateFormat)}" should be less than ${format(maxDate, dateFormat)}`;

if (errorMessage == null) return true;
return new PanelError(`Value out of range`, null, errorMessage);
};

const isValid = (dataStr, type, options, canBeEmpty = false) => {
if (canBeEmpty && dataStr === '') {
return true;
}
switch (type) {
case 'bool':
return isBool(dataStr);
case 'date':
if (!options.dateFormat) {
console.error("Missing option dateFormat, can't perform date validation.");
return false;
}
return isDate(dataStr, options.dateFormat);
return isBool(dataStr) || forgeTypeError(dataStr, type, options);
case 'date': {
if (!options?.dateFormat) return forgeConfigError("Missing option dateFormat, can't perform date validation.");
if (!isDate(dataStr, options?.dateFormat)) return forgeTypeError(dataStr, type, options);
const valueAsDate = DateUtils.parse(dataStr, options?.dateFormat);
return isDateInRange(valueAsDate, options?.minValue, options?.maxValue, options?.dateFormat);
}
case 'enum':
if (!options.enumValues) {
console.error("Missing option enumValues, can't perform enum validation.");
return false;
}
return isEnum(dataStr, options.enumValues);
if (!options.enumValues) return forgeConfigError("Missing option enumValues, can't perform enum validation.");
return isEnum(dataStr, options.enumValues) || forgeTypeError(dataStr, type, options);
case 'int':
return isInt(dataStr);
if (!isInt(dataStr)) return forgeTypeError(dataStr, type, options);
return isInRange(Number(dataStr), options?.minValue, options?.maxValue);
case 'number':
return isNumber(dataStr);
if (!isNumber(dataStr)) return forgeTypeError(dataStr, type, options);
return isInRange(Number(dataStr), options?.minValue, options?.maxValue);
case 'string':
return isString(dataStr);
return isString(dataStr) || forgeTypeError(dataStr, type, options);
default:
console.error(`Unknown type "${type}", can't perform type validation.`);
return false;
return forgeConfigError(`Unknown type "${type}", can't perform type validation.`);
}
};

Expand Down
Loading
Loading