Skip to content

Commit 6a056d2

Browse files
thisismayureshthisismayuresh
authored
feat: Added the isInvalidRecords in the AutoImport RSS XML Webhook Response Data (#1071)
2 parents 1555bad + 879b8c9 commit 6a056d2

File tree

6 files changed

+309
-13
lines changed

6 files changed

+309
-13
lines changed

‎apps/queue-manager/src/consumers/get-import-job-data.consumer.ts‎

Lines changed: 289 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,25 @@
11
import { RSSXMLService } from '@impler/services';
2-
import { ImportJobHistoryStatusEnum, SendImportJobCachedData } from '@impler/shared';
3-
import { JobMappingRepository, CommonRepository } from '@impler/dal';
2+
import {
3+
ImportJobHistoryStatusEnum,
4+
SendImportJobCachedData,
5+
ColumnTypesEnum,
6+
ITemplateSchemaItem,
7+
ColumnDelimiterEnum,
8+
} from '@impler/shared';
9+
410
import { SendImportJobDataConsumer } from './send-import-job-data.consumer';
11+
import { CommonRepository, JobMappingRepository, ColumnRepository } from '@impler/dal';
512

613
export class GetImportJobDataConsumer extends SendImportJobDataConsumer {
714
private commonRepository: CommonRepository = new CommonRepository();
815
private jobMappingRepository: JobMappingRepository = new JobMappingRepository();
16+
private columnRepo: ColumnRepository = new ColumnRepository();
917
private rssXmlService: RSSXMLService = new RSSXMLService();
1018

1119
async message(message: { content: string }) {
1220
const data = JSON.parse(message.content) as { _jobId: string };
1321
const importJobHistoryId = this.commonRepository.generateMongoId().toString();
14-
const importedData = await this.getJobImportedData(data._jobId);
22+
const validationResult = await this.getJobImportedData(data._jobId);
1523

1624
// Create history entry
1725
await this.importJobHistoryRepository.create({
@@ -26,13 +34,26 @@ export class GetImportJobDataConsumer extends SendImportJobDataConsumer {
2634
});
2735

2836
if (webhookDestination?.callbackUrl) {
29-
await this.sendDataImportData(data._jobId, importedData);
37+
if (validationResult.validRecords > 0) {
38+
await this.sendDataImportData(data._jobId, validationResult.validData, 1, undefined, false);
39+
}
40+
if (validationResult.invalidRecords > 0) {
41+
await this.sendDataImportData(data._jobId, validationResult.invalidData, 1, undefined, true);
42+
}
3043
}
3144

3245
return;
3346
}
3447

35-
async getJobImportedData(_jobId: string) {
48+
async getJobImportedData(_jobId: string): Promise<{
49+
importedData: Record<string, unknown>[];
50+
hasInvalidRecords: boolean;
51+
totalRecords: number;
52+
validRecords: number;
53+
invalidRecords: number;
54+
validData: Record<string, unknown>[];
55+
invalidData: Record<string, unknown>[];
56+
}> {
3657
try {
3758
const userJob = await this.userJobRepository.findOne({ _id: _jobId });
3859
if (!userJob) {
@@ -55,17 +76,144 @@ export class GetImportJobDataConsumer extends SendImportJobDataConsumer {
5576
const batchResult = await this.rssXmlService.getBatchXMLKeyValuesByPaths(parsedXMLData.xmlData, mappings);
5677
const mappedData = await this.rssXmlService.mappingFunction(mappings, batchResult);
5778

58-
return mappedData;
79+
const validationResult = await this.validateData(_jobId, mappedData);
80+
81+
// Send data to webhook with validation status
82+
this.sendDataImportData(_jobId, mappedData, 1, undefined, validationResult.hasInvalidRecords);
83+
84+
if (validationResult.hasInvalidRecords) {
85+
await this.userJobRepository.update({ _id: _jobId }, { $set: { isInvalidRecords: true } });
86+
}
87+
88+
return {
89+
importedData: mappedData,
90+
hasInvalidRecords: validationResult.hasInvalidRecords,
91+
totalRecords: validationResult.totalRecords,
92+
validRecords: validationResult.validRecords,
93+
invalidRecords: validationResult.invalidRecords,
94+
validData: validationResult.validData,
95+
invalidData: validationResult.invalidData,
96+
};
5997
} catch (error) {
6098
throw error;
6199
}
62100
}
63101

102+
private async validateData(
103+
_jobId: string,
104+
mappedData: Record<string, unknown>[]
105+
): Promise<{
106+
hasInvalidRecords: boolean;
107+
totalRecords: number;
108+
validRecords: number;
109+
invalidRecords: number;
110+
validData: Record<string, unknown>[];
111+
invalidData: Record<string, unknown>[];
112+
}> {
113+
try {
114+
const userJob = await this.userJobRepository.findOne({ _id: _jobId });
115+
if (!userJob) {
116+
throw new Error(`Job not found for _jobId: ${_jobId}`);
117+
}
118+
119+
// Get template columns (schema)
120+
const columns = await this.columnRepo.find({ _templateId: userJob._templateId });
121+
if (!columns || columns.length === 0) {
122+
return {
123+
hasInvalidRecords: false,
124+
totalRecords: 0,
125+
validRecords: 0,
126+
invalidRecords: 0,
127+
validData: [],
128+
invalidData: [],
129+
};
130+
}
131+
132+
const multiSelectColumnHeadings: Record<string, string> = {};
133+
134+
(columns as unknown as ITemplateSchemaItem[]).forEach((column) => {
135+
if (column.type === ColumnTypesEnum.SELECT && column.allowMultiSelect)
136+
multiSelectColumnHeadings[column.key] = column.delimiter || ColumnDelimiterEnum.COMMA;
137+
});
138+
139+
let totalRecords = 0;
140+
let validRecords = 0;
141+
let invalidRecords = 0;
142+
const validData: Record<string, unknown>[] = [];
143+
const invalidData: Record<string, unknown>[] = [];
144+
145+
for (const recordData of mappedData) {
146+
// Format record for multi-select handling
147+
const checkRecord: Record<string, unknown> = this.formatRecord({
148+
record: { record: recordData },
149+
multiSelectColumnHeadings,
150+
});
151+
152+
const validationResult = this.validateRecordUsingColumnSchema(
153+
checkRecord,
154+
columns as unknown as ITemplateSchemaItem[]
155+
);
156+
157+
totalRecords++;
158+
159+
if (validationResult.isValid) {
160+
validRecords++;
161+
validData.push(recordData);
162+
} else {
163+
invalidRecords++;
164+
// Include validation errors with the invalid record
165+
invalidData.push(recordData);
166+
}
167+
}
168+
169+
const hasInvalidRecords = invalidRecords > 0;
170+
171+
return {
172+
hasInvalidRecords,
173+
totalRecords,
174+
validRecords,
175+
invalidRecords,
176+
validData,
177+
invalidData,
178+
};
179+
} catch (error) {
180+
return {
181+
hasInvalidRecords: false,
182+
totalRecords: 0,
183+
validRecords: 0,
184+
invalidRecords: 0,
185+
validData: [],
186+
invalidData: [],
187+
};
188+
}
189+
}
190+
191+
// Format record method from ReReviewData
192+
private formatRecord({
193+
record,
194+
multiSelectColumnHeadings,
195+
}: {
196+
record: { record: Record<string, unknown> };
197+
multiSelectColumnHeadings?: Record<string, string>;
198+
}) {
199+
return Object.keys(multiSelectColumnHeadings || {}).reduce(
200+
(acc, heading) => {
201+
if (typeof record.record[heading] === 'string') {
202+
acc[heading] = (record.record[heading] as string)?.split(multiSelectColumnHeadings[heading]);
203+
}
204+
205+
return acc;
206+
},
207+
{ ...record.record }
208+
);
209+
}
210+
64211
private async sendDataImportData(
65212
_jobId: string,
66-
allDataJson: any[],
213+
allDataJson: Record<string, any>[],
67214
page = 1,
68-
initialCachedData?: SendImportJobCachedData
215+
initialCachedData?: SendImportJobCachedData,
216+
areInvalidRecords?: boolean
69217
) {
70218
try {
71219
let cachedData = null;
@@ -84,6 +232,7 @@ export class GetImportJobDataConsumer extends SendImportJobDataConsumer {
84232
recordFormat: cachedData.recordFormat,
85233
chunkFormat: cachedData.chunkFormat,
86234
...cachedData,
235+
isInvalidRecords: areInvalidRecords,
87236
});
88237

89238
const headers =
@@ -109,15 +258,145 @@ export class GetImportJobDataConsumer extends SendImportJobDataConsumer {
109258
});
110259

111260
if (nextPageNumber) {
112-
// Recursively call for next page with updated page number
113261
await this.sendDataImportData(_jobId, allDataJson, nextPageNumber, { ...cachedData, page: nextPageNumber });
114262
} else {
115-
// Processing is done
116263
await this.finalizeUpload(_jobId);
117264
}
118265
}
119266
} catch (error) {
120267
throw error;
121268
}
122269
}
270+
271+
private validateRecordUsingColumnSchema(
272+
record: Record<string, unknown>,
273+
columns: ITemplateSchemaItem[]
274+
): { isValid: boolean; errors: Record<string, string> } {
275+
enum ValidationTypesEnum {
276+
RANGE = 'range',
277+
LENGTH = 'length',
278+
UNIQUE_WITH = 'unique_with',
279+
DIGITS = 'digits',
280+
}
281+
const errors: Record<string, string> = {};
282+
let isValid = true;
283+
284+
for (const column of columns) {
285+
const value = record[column.key];
286+
287+
if (value === undefined) {
288+
errors[column.key] = `${column.key} has undefined value`;
289+
isValid = false;
290+
continue;
291+
}
292+
293+
if (column.isRequired && (value === null || value === '' || !value)) {
294+
errors[column.key] = `${column.key} is required`;
295+
isValid = false;
296+
continue;
297+
}
298+
299+
if (value !== null && value !== '') {
300+
switch (column.type) {
301+
case ColumnTypesEnum.NUMBER:
302+
if (isNaN(Number(value))) {
303+
errors[column.key] = `${column.key} must be a valid number`;
304+
isValid = false;
305+
}
306+
break;
307+
case ColumnTypesEnum.DOUBLE:
308+
if (isNaN(Number(value)) || !Number.isFinite(Number(value))) {
309+
errors[column.key] = `${column.key} must be a valid decimal number`;
310+
isValid = false;
311+
}
312+
break;
313+
case ColumnTypesEnum.EMAIL:
314+
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
315+
if (!emailRegex.test(String(value))) {
316+
errors[column.key] = `${column.key} must be a valid email address`;
317+
isValid = false;
318+
}
319+
break;
320+
case ColumnTypesEnum.DATE:
321+
if (isNaN(Date.parse(String(value)))) {
322+
errors[column.key] = `${column.key} must be a valid date`;
323+
isValid = false;
324+
}
325+
break;
326+
case ColumnTypesEnum.REGEX:
327+
if (column.regex && !new RegExp(column.regex).test(String(value))) {
328+
errors[column.key] = `${column.key} does not match required format`;
329+
isValid = false;
330+
}
331+
break;
332+
case ColumnTypesEnum.SELECT:
333+
if (column.selectValues && !column.selectValues.includes(String(value))) {
334+
errors[column.key] = `${column.key} must be one of: ${column.selectValues.join(', ')}`;
335+
isValid = false;
336+
}
337+
break;
338+
case ColumnTypesEnum.IMAGE:
339+
const imageUrlRegex = /^https?:\/\/.+\.(jpg|jpeg|png|gif|bmp|webp|svg)$/i;
340+
if (!imageUrlRegex.test(String(value))) {
341+
errors[column.key] = `${column.key} must be a valid image URL`;
342+
isValid = false;
343+
}
344+
break;
345+
}
346+
}
347+
348+
if (
349+
value !== undefined &&
350+
value !== null &&
351+
value !== '' &&
352+
column.validations &&
353+
column.validations.length > 0
354+
) {
355+
for (const validation of column.validations) {
356+
switch (validation.validate) {
357+
case ValidationTypesEnum.RANGE:
358+
const numValue = Number(value);
359+
if (!isNaN(numValue)) {
360+
if (validation.min !== undefined && numValue < validation.min) {
361+
errors[column.key] = `${column.key} must be at least ${validation.min}`;
362+
isValid = false;
363+
}
364+
if (validation.max !== undefined && numValue > validation.max) {
365+
errors[column.key] = `${column.key} must be at most ${validation.max}`;
366+
isValid = false;
367+
}
368+
}
369+
break;
370+
case ValidationTypesEnum.LENGTH:
371+
const strValue = String(value);
372+
if (validation.min !== undefined && strValue.length < validation.min) {
373+
errors[column.key] = `${column.key} must be at least ${validation.min} characters long`;
374+
isValid = false;
375+
}
376+
if (validation.max !== undefined && strValue.length > validation.max) {
377+
errors[column.key] = `${column.key} must be at most ${validation.max} characters long`;
378+
isValid = false;
379+
}
380+
break;
381+
case ValidationTypesEnum.DIGITS:
382+
const digitStr = String(value).replace(/[^0-9]/g, '');
383+
if (validation.min !== undefined && digitStr.length < validation.min) {
384+
errors[column.key] = `${column.key} must have at least ${validation.min} digits`;
385+
isValid = false;
386+
}
387+
if (validation.max !== undefined && digitStr.length > validation.max) {
388+
errors[column.key] = `${column.key} must have at most ${validation.max} digits`;
389+
isValid = false;
390+
}
391+
break;
392+
case ValidationTypesEnum.UNIQUE_WITH:
393+
break;
394+
}
395+
if (!isValid) break;
396+
}
397+
}
398+
}
399+
400+
return { isValid, errors };
401+
}
123402
}

‎apps/queue-manager/src/consumers/send-import-job-data.consumer.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,15 @@ export class SendImportJobDataConsumer extends BaseConsumer {
104104
recordFormat,
105105
extra = '',
106106
multiSelectHeadings,
107-
}: SendImportJobCachedData & { data: any[]; uploadId: string }): { sendData: Record<string, unknown>; page: number } {
107+
isInvalidRecords,
108+
}: SendImportJobCachedData & {
109+
data: any[];
110+
uploadId: string;
111+
isInvalidRecords?: boolean;
112+
}): {
113+
sendData: Record<string, unknown>;
114+
page: number;
115+
} {
108116
const defaultValuesObj = JSON.parse(defaultValues);
109117
let slicedData = data.slice(
110118
Math.max((page - DEFAULT_PAGE) * chunkSize, MIN_LIMIT),
@@ -132,6 +140,7 @@ export class SendImportJobDataConsumer extends BaseConsumer {
132140
chunkSize: slicedData.length,
133141
extra: extra ? JSON.parse(extra) : '',
134142
totalPages: this.getTotalPages(data.length, chunkSize),
143+
...(isInvalidRecords !== undefined && { isInvalidRecords }),
135144
};
136145

137146
return {
@@ -182,6 +191,7 @@ export class SendImportJobDataConsumer extends BaseConsumer {
182191
defaultValues: JSON.stringify(defaultValueObj),
183192
recordFormat: userJob.customRecordFormat,
184193
chunkFormat: userJob.customChunkFormat,
194+
isInvalidRecords: userJob.isInvalidRecords,
185195
};
186196
}
187197

0 commit comments

Comments
 (0)