Skip to content

Commit 5a57bb0

Browse files
thisismayureshthisismayuresh
committed
feat: Added the isInvalidRecords in the AutoImport RSS XML Webhook Data
1 parent 1555bad commit 5a57bb0

File tree

6 files changed

+149
-8
lines changed

6 files changed

+149
-8
lines changed

‎apps/queue-manager/src/consumers/get-import-job-data.consumer.ts‎

Lines changed: 129 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
import { RSSXMLService } from '@impler/services';
2-
import { ImportJobHistoryStatusEnum, SendImportJobCachedData } from '@impler/shared';
3-
import { JobMappingRepository, CommonRepository } from '@impler/dal';
2+
import { ImportJobHistoryStatusEnum, SendImportJobCachedData, ColumnTypesEnum } from '@impler/shared';
3+
import { JobMappingRepository, CommonRepository, ColumnRepository } from '@impler/dal';
44
import { SendImportJobDataConsumer } from './send-import-job-data.consumer';
55

66
export class GetImportJobDataConsumer extends SendImportJobDataConsumer {
77
private commonRepository: CommonRepository = new CommonRepository();
88
private jobMappingRepository: JobMappingRepository = new JobMappingRepository();
9+
private columnRepo: ColumnRepository = new ColumnRepository();
910
private rssXmlService: RSSXMLService = new RSSXMLService();
1011

1112
async message(message: { content: string }) {
@@ -32,7 +33,7 @@ export class GetImportJobDataConsumer extends SendImportJobDataConsumer {
3233
return;
3334
}
3435

35-
async getJobImportedData(_jobId: string) {
36+
async getJobImportedData(_jobId: string): Promise<Record<string, any>[]> {
3637
try {
3738
const userJob = await this.userJobRepository.findOne({ _id: _jobId });
3839
if (!userJob) {
@@ -55,15 +56,138 @@ export class GetImportJobDataConsumer extends SendImportJobDataConsumer {
5556
const batchResult = await this.rssXmlService.getBatchXMLKeyValuesByPaths(parsedXMLData.xmlData, mappings);
5657
const mappedData = await this.rssXmlService.mappingFunction(mappings, batchResult);
5758

58-
return mappedData;
59+
const hasInvalidRecords = await this.validateWithTemplateColumns(_jobId, mappedData);
60+
61+
if (hasInvalidRecords) {
62+
await this.userJobRepository.update({ _id: _jobId }, { $set: { isInvalidRecords: true } });
63+
}
64+
65+
return mappedData as Record<string, any>[];
5966
} catch (error) {
6067
throw error;
6168
}
6269
}
6370

71+
private async validateWithTemplateColumns(_jobId: string, mappedData: Record<string, any>[]): Promise<boolean> {
72+
enum ValidationTypesEnum {
73+
RANGE = 'range',
74+
LENGTH = 'length',
75+
UNIQUE_WITH = 'unique_with',
76+
DIGITS = 'digits',
77+
}
78+
79+
try {
80+
const userJob = await this.userJobRepository.findOne({ _id: _jobId });
81+
const columns = await this.columnRepo.find({ _templateId: userJob._templateId });
82+
83+
if (!columns || columns.length === 0) {
84+
return false;
85+
}
86+
87+
let invalidRecordCount = 0;
88+
const totalRecords = mappedData.length;
89+
90+
for (const record of mappedData) {
91+
let recordHasErrors = false;
92+
93+
for (const column of columns) {
94+
const value = record[column.key];
95+
96+
if (column.isRequired && (!value || value === '' || value === null)) {
97+
recordHasErrors = true;
98+
break;
99+
}
100+
101+
if (value && value !== '') {
102+
switch (column.type) {
103+
case ColumnTypesEnum.NUMBER:
104+
if (isNaN(Number(value))) recordHasErrors = true;
105+
break;
106+
case ColumnTypesEnum.DOUBLE:
107+
if (isNaN(Number(value)) || !Number.isFinite(Number(value))) recordHasErrors = true;
108+
break;
109+
case ColumnTypesEnum.EMAIL:
110+
const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
111+
if (!emailRegex.test(value)) recordHasErrors = true;
112+
break;
113+
case ColumnTypesEnum.DATE:
114+
if (isNaN(Date.parse(value))) recordHasErrors = true;
115+
break;
116+
case ColumnTypesEnum.REGEX:
117+
if (column.regex && !new RegExp(column.regex).test(value)) {
118+
recordHasErrors = true;
119+
}
120+
break;
121+
case ColumnTypesEnum.SELECT:
122+
if (column.selectValues && !column.selectValues.includes(value)) {
123+
recordHasErrors = true;
124+
}
125+
break;
126+
case ColumnTypesEnum.IMAGE:
127+
// Basic URL validation for images
128+
const imageUrlRegex = /^https?:\/\/.+\.(jpg|jpeg|png|gif|bmp|webp|svg)$/i;
129+
if (!imageUrlRegex.test(value)) recordHasErrors = true;
130+
break;
131+
}
132+
}
133+
134+
if (!recordHasErrors && column.validations && column.validations.length > 0) {
135+
for (const validation of column.validations) {
136+
switch (validation.validate) {
137+
case ValidationTypesEnum.RANGE:
138+
const numValue = Number(value);
139+
if (!isNaN(numValue)) {
140+
if (validation.min !== undefined && numValue < validation.min) {
141+
recordHasErrors = true;
142+
}
143+
if (validation.max !== undefined && numValue > validation.max) {
144+
recordHasErrors = true;
145+
}
146+
}
147+
break;
148+
case ValidationTypesEnum.LENGTH:
149+
const strValue = String(value);
150+
if (validation.min !== undefined && strValue.length < validation.min) {
151+
recordHasErrors = true;
152+
}
153+
if (validation.max !== undefined && strValue.length > validation.max) {
154+
recordHasErrors = true;
155+
}
156+
break;
157+
case ValidationTypesEnum.DIGITS:
158+
const digitStr = String(value).replace(/[^0-9]/g, '');
159+
if (validation.min !== undefined && digitStr.length < validation.min) {
160+
recordHasErrors = true;
161+
}
162+
if (validation.max !== undefined && digitStr.length > validation.max) {
163+
recordHasErrors = true;
164+
}
165+
break;
166+
case ValidationTypesEnum.UNIQUE_WITH:
167+
break;
168+
}
169+
if (recordHasErrors) break;
170+
}
171+
}
172+
173+
if (recordHasErrors) break;
174+
}
175+
176+
if (recordHasErrors) {
177+
invalidRecordCount++;
178+
}
179+
}
180+
181+
// Consider batch invalid if >30% of records have validation errors
182+
return totalRecords > 0 && invalidRecordCount / totalRecords > 0.3;
183+
} catch (error) {
184+
return false;
185+
}
186+
}
187+
64188
private async sendDataImportData(
65189
_jobId: string,
66-
allDataJson: any[],
190+
allDataJson: Record<string, any>[],
67191
page = 1,
68192
initialCachedData?: SendImportJobCachedData
69193
) {

‎apps/queue-manager/src/consumers/send-import-job-data.consumer.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,15 @@ export class SendImportJobDataConsumer extends BaseConsumer {
104104
recordFormat,
105105
extra = '',
106106
multiSelectHeadings,
107-
}: SendImportJobCachedData & { data: any[]; uploadId: string }): { sendData: Record<string, unknown>; page: number } {
107+
isInvalidRecords,
108+
}: SendImportJobCachedData & {
109+
data: any[];
110+
uploadId: string;
111+
isInvalidRecords?: boolean;
112+
}): {
113+
sendData: Record<string, unknown>;
114+
page: number;
115+
} {
108116
const defaultValuesObj = JSON.parse(defaultValues);
109117
let slicedData = data.slice(
110118
Math.max((page - DEFAULT_PAGE) * chunkSize, MIN_LIMIT),
@@ -132,6 +140,7 @@ export class SendImportJobDataConsumer extends BaseConsumer {
132140
chunkSize: slicedData.length,
133141
extra: extra ? JSON.parse(extra) : '',
134142
totalPages: this.getTotalPages(data.length, chunkSize),
143+
...(isInvalidRecords !== undefined && { isInvalidRecords }),
135144
};
136145

137146
return {
@@ -182,6 +191,7 @@ export class SendImportJobDataConsumer extends BaseConsumer {
182191
defaultValues: JSON.stringify(defaultValueObj),
183192
recordFormat: userJob.customRecordFormat,
184193
chunkFormat: userJob.customChunkFormat,
194+
isInvalidRecords: userJob.isInvalidRecords,
185195
};
186196
}
187197

‎libs/dal/src/repositories/user-job/user-job.entity.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,6 @@ export class UserJobEntity {
2626
customChunkFormat: string;
2727

2828
customSchema: string;
29+
30+
isInvalidRecords?: boolean;
2931
}

‎libs/dal/src/repositories/user-job/user-job.schema.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,10 @@ const userJobSchema = new Schema(
4646
customSchema: {
4747
type: Schema.Types.String,
4848
},
49+
isInvalidRecords: {
50+
type: Schema.Types.Boolean,
51+
default: false,
52+
},
4953
},
5054
{ ...schemaOptions }
5155
);

‎libs/services/src/rss-xml/rssxml.service.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ export class RSSXMLService {
115115
// Parse paths and prepare for extraction
116116
const parsedPaths = pathMappings.map((mapping) => ({
117117
key: mapping.key,
118-
pathArray: mapping.path.split('>').map((key) => key.trim()),
118+
pathArray: mapping.path?.split('>').map((key) => key.trim()) || [],
119119
originalPath: mapping.path,
120120
}));
121121

@@ -500,7 +500,7 @@ export class RSSXMLService {
500500
async setValue(obj: Record<string, any>, path: string[], value: any, attributes?: any): Promise<void> {
501501
// Validate path to prevent prototype pollution
502502
const forbiddenKeys = ['__proto__', 'constructor', 'prototype'];
503-
if (path.some(key => forbiddenKeys.includes(key))) {
503+
if (path.some((key) => forbiddenKeys.includes(key))) {
504504
throw new Error('Invalid path: contains forbidden keys');
505505
}
506506

‎libs/shared/src/types/upload/upload.types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ export type CommonCachedData = {
7070
defaultValues: string;
7171
multiSelectHeadings?: Record<string, string>;
7272
imageHeadings?: string[];
73+
isInvalidRecords?: boolean;
7374
};
7475

7576
export type SendWebhookCachedData = {

0 commit comments

Comments
 (0)