Skip to content

Display error msg when a non-visible character is present in the csv #1200

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
joshunrau merged 29 commits into from
Oct 9, 2025
Merged

Display error msg when a non-visible character is present in the csv #1200

Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
0785071
feat: implement described error msg for sample data/subject row conta…
david-roper Sep 23, 2025
ee6bb98
feat: new error check for non-visible chars in non-initial rows
david-roper Sep 23, 2025
7941210
feat: non-visible character check in zod3 processInstrumentCSV method…
david-roper Sep 23, 2025
b376302
fix: remove replacements methods as error should cover it
david-roper Sep 23, 2025
f7d7417
fix: adjust logic for non-visible char check, fix typo
david-roper Sep 23, 2025
7319c38
fix: edits to french error messages
david-roper Sep 24, 2025
c2b2dd2
fix: remove false return value and logic using it
david-roper Sep 24, 2025
8b0bbdb
fix: refactor if statements and error msg
david-roper Sep 24, 2025
28166fa
chore: fix quotation
david-roper Sep 24, 2025
75b7257
chore: fix captialization of error messages
david-roper Sep 26, 2025
47e8147
Update apps/web/src/utils/upload.ts
david-roper Sep 26, 2025
e8088b5
fix: improve regex matching, update error msgs to contain matched chars
david-roper Sep 29, 2025
5112d4e
fix: add s to nonvisiblechars variable
david-roper Sep 29, 2025
a337ee8
fix: typo
david-roper Sep 29, 2025
4b434cd
feat: add coderabit regex recommendation
david-roper Sep 29, 2025
26b9861
feat: display unicode value when caught
david-roper Sep 30, 2025
870faac
chore: test new regex
david-roper Oct 2, 2025
4eae3c7
chore: use regex from printable chars js
david-roper Oct 3, 2025
92700e3
test: fix type casting issues in upload test
david-roper Oct 3, 2025
29bdc37
fix: formatting
david-roper Oct 3, 2025
3ddadaa
chore: move regex variable
david-roper Oct 3, 2025
90c6ab3
Update apps/web/src/utils/upload.ts
david-roper Oct 6, 2025
ed5eb49
chore: fix code rabbit suggestion
david-roper Oct 6, 2025
c5b9005
chore: unicode non visible chars in initial row and date check
david-roper Oct 6, 2025
0ce61a8
chore: add mongolian vowel separator unicode to regex catch
david-roper Oct 6, 2025
10e8c78
chore: add tab to regex catcher
david-roper Oct 6, 2025
e8292c9
feat: deal with 3 spaces (tab in vscode csv editor)
david-roper Oct 8, 2025
553cee9
Update apps/web/src/utils/upload.ts
david-roper Oct 8, 2025
e81547a
chore: changed my mind spaces now allowed
david-roper Oct 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 21 additions & 21 deletions apps/web/src/utils/__tests__/upload.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ describe('Zod3', () => {
});

it('should parse array of objects', () => {
const result = Zod3.getZodTypeName(z3.array(z3.object({ name: z3.string(), age: z3.number() })));
const result = Zod3.getZodTypeName(z3.array(z3.object({ age: z3.number(), name: z3.string() })));
expect(result).toMatchObject({
isOptional: false,
multiKeys: ['name', 'age'],
Expand All @@ -58,10 +58,10 @@ describe('Zod3', () => {
describe('processInstrumentCSV', () => {
const mockInstrument = {
validationSchema: z3.object({
score: z3.number(),
notes: z3.string()
notes: z3.string(),
score: z3.number()
})
} as AnyUnilingualFormInstrument;
} as unknown as AnyUnilingualFormInstrument;

it('should process valid CSV data', async () => {
const csvContent = unparse([
Expand All @@ -74,9 +74,9 @@ describe('Zod3', () => {

expect(result).toHaveLength(1);
expect(result[0]).toMatchObject({
subjectID: 'subject1',
notes: 'Good performance',
score: 85,
notes: 'Good performance'
subjectID: 'subject1'
});
});

Expand All @@ -89,10 +89,10 @@ describe('Zod3', () => {
it('should handle optional fields', async () => {
const instrumentWithOptional = {
validationSchema: z3.object({
required: z3.string(),
optional: z3.string().optional()
optional: z3.string().optional(),
required: z3.string()
})
} as AnyUnilingualFormInstrument;
} as unknown as AnyUnilingualFormInstrument;

const csvContent = unparse([
['subjectID', 'date', 'required', 'optional'],
Expand All @@ -104,8 +104,8 @@ describe('Zod3', () => {

expect(result).toHaveLength(1);
expect(result[0]).toMatchObject({
required: 'value',
optional: undefined
optional: undefined,
required: 'value'
});
});

Expand All @@ -114,7 +114,7 @@ describe('Zod3', () => {
validationSchema: z3.object({
completed: z3.boolean()
})
} as AnyUnilingualFormInstrument;
} as unknown as AnyUnilingualFormInstrument;

const csvContent = unparse([
['subjectID', 'date', 'completed'],
Expand All @@ -133,7 +133,7 @@ describe('Zod3', () => {
validationSchema: z3.object({
tags: z3.set(z3.enum(['tag1', 'tag2', 'tag3']))
})
} as AnyUnilingualFormInstrument;
} as unknown as AnyUnilingualFormInstrument;

const csvContent = unparse([
['subjectID', 'date', 'tags'],
Expand Down Expand Up @@ -197,10 +197,10 @@ describe('Zod4', () => {
describe('processInstrumentCSV', () => {
const mockInstrument = {
validationSchema: z4.object({
score: z4.number(),
feedback: z4.string()
feedback: z4.string(),
score: z4.number()
})
} as AnyUnilingualFormInstrument;
} as unknown as AnyUnilingualFormInstrument;

it('should process valid CSV data', async () => {
const csvContent = unparse([
Expand All @@ -213,9 +213,9 @@ describe('Zod4', () => {

expect(result).toHaveLength(1);
expect(result[0]).toMatchObject({
subjectID: 'subject1',
feedback: 'Excellent work',
score: 92,
feedback: 'Excellent work'
subjectID: 'subject1'
});
});

Expand All @@ -234,7 +234,7 @@ describe('Zod4', () => {
validationSchema: z4.object({
eventDate: z4.date()
})
} as AnyUnilingualFormInstrument;
} as unknown as AnyUnilingualFormInstrument;

const csvContent = unparse([
['subjectID', 'date', 'eventDate'],
Expand All @@ -253,7 +253,7 @@ describe('Zod4', () => {
validationSchema: z4.object({
status: z4.enum(['pending', 'active', 'completed'])
})
} as AnyUnilingualFormInstrument;
} as unknown as AnyUnilingualFormInstrument;

const csvContent = unparse([
['subjectID', 'date', 'status'],
Expand All @@ -277,7 +277,7 @@ describe('Zod4', () => {
})
)
})
} as AnyUnilingualFormInstrument;
} as unknown as AnyUnilingualFormInstrument;

const csvContent = unparse([
['subjectID', 'date', 'items'],
Expand Down
115 changes: 103 additions & 12 deletions apps/web/src/utils/upload.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,26 @@ function parseSetEntry(entry: string): Set<string> {
return set;
}

const ansiEscapeCode = '[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-PRZcf-nqry=><]',
zeroWidthCharacterExceptNewline =
'\u0000-\u0009\u000B-\u0019\u001b\u180e\u009b\u00ad\u200b\u2028\u2029\ufeff\ufe00-\ufe0f';

const zeroWidthCharactersExceptNewline = new RegExp(
// eslint-disable-next-line no-misleading-character-class
'(?:' + ansiEscapeCode + ')|[' + zeroWidthCharacterExceptNewline + ']|(?: {3})',
'g'
);

function nonVisibleCharChecker(entry: string | undefined) {
if (!entry) {
return null;
}

zeroWidthCharactersExceptNewline.lastIndex = 0;
const nonVisibleCharCheck = zeroWidthCharactersExceptNewline.exec(entry);
return nonVisibleCharCheck;
}

const ZOD_TYPE_NAMES = [
'ZodNumber',
'ZodString',
Expand Down Expand Up @@ -506,17 +526,40 @@ export namespace Zod3 {
);
}

let rowNumber = 1;

const regexResultSubject = nonVisibleCharChecker(dataLines[0][0]);
const regexResultDate = nonVisibleCharChecker(dataLines[0][1]);

if (regexResultSubject !== null) {
const charCode = regexResultSubject[0].charCodeAt(0).toString(16).toUpperCase().padStart(4, '0');
return reject(
new UploadError({
en: `Subject ID at row ${rowNumber} contains non-visible character(s) (U+${charCode})`,
fr: `L'ID du sujet à la ligne ${rowNumber} contient des caractères non visible(s) (U+${charCode})`
})
);
}
if (regexResultDate !== null) {
const charCode = regexResultDate[0].charCodeAt(0).toString(16).toUpperCase().padStart(4, '0');
return reject(
new UploadError({
en: `Date at row ${rowNumber} contains non-visible character(s) (U+${charCode})`,
fr: `Date à la ligne ${rowNumber} contient des caractères non visible(s) (U+${charCode})`
})
);
}

//remove sample data if included remove any mongolian vowel separators
if (
dataLines[0][0]?.replace(/[\u200B-\u200D\uFEFF\u180E]/g, '').trim() === INTERNAL_HEADERS_SAMPLE_DATA[0] &&
dataLines[0][1]?.replace(/[\u200B-\u200D\uFEFF\u180E]/g, '').trim() === INTERNAL_HEADERS_SAMPLE_DATA[1]
dataLines[0][0]?.trim() === INTERNAL_HEADERS_SAMPLE_DATA[0] &&
dataLines[0][1]?.trim() === INTERNAL_HEADERS_SAMPLE_DATA[1]
) {
dataLines.shift();
}

const result: FormTypes.Data[] = [];

let rowNumber = 1;
for (const elements of dataLines) {
const jsonLine: { [key: string]: unknown } = {};
for (let i = 0; i < headers.length; i++) {
Expand All @@ -525,6 +568,19 @@ export namespace Zod3 {
if (rawValue === '\n') {
continue;
}

//Check for non visible char in every row, return error if present
const nonVisibleChars = nonVisibleCharChecker(rawValue);
if (nonVisibleChars !== null) {
const charCode = nonVisibleChars[0].charCodeAt(0).toString(16).toUpperCase().padStart(4, '0');
return reject(
new UploadError({
en: `Value at row ${rowNumber} and column '${key}' contains non-visible character(s) (U+${charCode})`,
fr: `La valeur à la ligne ${rowNumber} et colonne '${key}' contient des caractère(s) non visibles (U+${charCode})`
})
);
}

if (shape[key] === undefined) {
return reject(
new UploadError({
Expand All @@ -541,7 +597,7 @@ export namespace Zod3 {
return reject(
new UploadError({
en: `${error.description.en} at column name: '${key}' and row number '${rowNumber}'`,
fr: `${error.description.fr} au nom de colonne : '${key}' et numéro de ligne '${rowNumber}`
fr: `${error.description.fr} au nom de colonne : '${key}' et numéro de ligne '${rowNumber}'`
})
);
}
Expand All @@ -560,7 +616,8 @@ export namespace Zod3 {
console.error(`Failed to parse data: ${JSON.stringify(jsonLine)}`);
return reject(
new UploadError({
en: 'Schema parsing failed: refer to the browser console for further details'
en: 'Schema parsing failed: refer to the browser console for further details',
fr: `Échec de l'analyse du schéma : reportez-vous à la console du navigateur pour plus de détails`
})
);
}
Expand Down Expand Up @@ -824,24 +881,58 @@ export namespace Zod4 {
}

//remove sample data if included (account for old mongolian vowel separator templates)
//return an error if non space characters are found

let rowNumber = 1;

const regexResultSubject = nonVisibleCharChecker(dataLines[0][0]);
const regexResultDate = nonVisibleCharChecker(dataLines[0][1]);

if (regexResultSubject !== null) {
const charCode = regexResultSubject[0].charCodeAt(0).toString(16).toUpperCase().padStart(4, '0');
return reject(
new UploadError({
en: `Subject ID at row ${rowNumber} contains non-visible characters (U+${charCode})`,
fr: `L'ID du sujet à la ligne ${rowNumber} contient des caractères non visibles (U+${charCode})`
})
);
}
if (regexResultDate !== null) {
const charCode = regexResultDate[0].charCodeAt(0).toString(16).toUpperCase().padStart(4, '0');
return reject(
new UploadError({
en: `Date at row ${rowNumber} contains non-visible characters (U+${charCode})`,
fr: `Date à la ligne ${rowNumber} contient des caractères non visibles (U+${charCode})`
})
);
}

if (
dataLines[0][0]?.replace(/[\u200B-\u200D\uFEFF\u180E]/g, '').trim() === INTERNAL_HEADERS_SAMPLE_DATA[0] &&
dataLines[0][1]?.replace(/[\u200B-\u200D\uFEFF\u180E]/g, '').trim() === INTERNAL_HEADERS_SAMPLE_DATA[1]
dataLines[0][0]?.trim() === INTERNAL_HEADERS_SAMPLE_DATA[0] &&
dataLines[0][1]?.trim() === INTERNAL_HEADERS_SAMPLE_DATA[1]
) {
dataLines.shift();
}

const result: FormTypes.Data[] = [];

let rowNumber = 1;
for (const elements of dataLines) {
const jsonLine: { [key: string]: unknown } = {};
for (let i = 0; i < headers.length; i++) {
const key = headers[i]!.trim();
const rawValue = elements[i]!.trim();
if (rawValue === '\n') {
continue;
const cell = elements[i];
const rawValue = cell == null ? '' : cell.trim();
if (rawValue === '\n') continue;
// Return error if any non‑visible character is present
const nonVisibleChars = nonVisibleCharChecker(rawValue);
if (nonVisibleChars !== null) {
const charCode = nonVisibleChars[0].charCodeAt(0).toString(16).toUpperCase().padStart(4, '0');
return reject(
new UploadError({
en: `Value at row ${rowNumber} and column '${key}' contains non-visible characters (U+${charCode})`,
fr: `La valeur à la ligne ${rowNumber} et colonne '${key}' contient des caractères non visibles (U+${charCode})`
})
);
}
if (shape[key] === undefined) {
return reject(
Expand All @@ -859,7 +950,7 @@ export namespace Zod4 {
return reject(
new UploadError({
en: `${error.description.en} at column name: '${key}' and row number '${rowNumber}'`,
fr: `${error.description.fr} au nom de colonne : '${key}' et numéro de ligne '${rowNumber}`
fr: `${error.description.fr} au nom de colonne : '${key}' et numéro de ligne '${rowNumber}'`
})
);
}
Expand Down
Loading