Skip to content

Commit 39e8c59

Browse files
thisismayureshthisismayuresh
authored
Merging Next (1.1.1) into ---> Main (#1023)
2 parents 21ae86f + 7c195bf commit 39e8c59

File tree

26 files changed

+7906
-4697
lines changed

26 files changed

+7906
-4697
lines changed

‎apps/api/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@impler/api",
3-
"version": "1.1.0",
3+
"version": "1.1.1",
44
"author": "implerhq",
55
"license": "MIT",
66
"private": true,

‎apps/api/src/app/auto-import-jobs-schedular/usecase/auto-import-jobs-schedular.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ export class AutoImportJobsSchedular {
1818
private readonly scheduleUserJob: ScheduleUserJob
1919
) {}
2020

21-
@Cron(CRON_SCHEDULE.DEFAULT_CRON_TIME)
21+
@Cron(CRON_SCHEDULE.AUTO_IMPORT_DEFAULT_CRON_TIME)
2222
async handleCronSchedular() {
2323
await this.fetchAndExecuteScheduledJobs();
2424
}

‎apps/api/src/app/shared/constants.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,9 +96,11 @@ export enum LEAD_SIGNUP_USING {
9696
}
9797

9898
export const CRON_SCHEDULE = {
99-
DEFAULT_CRON_TIME: '45 23 * * *',
99+
AUTO_IMPORT_DEFAULT_CRON_TIME: '45 23 * * *',
100+
UPLOAD_CLEANUP_DEFAULT_CRON_TIME: '30 22 * * *', //Cron - Every Night at 10:30 PM
100101
TEST_CRON_TIME: '* * * * *',
101102
CRON_TIME: '45 23 * * *',
103+
UPLOAD_CLEANUP_DAYS: 30,
102104
};
103105

104106
export const TYPE_CHECK_SAMPLE = 10;

‎apps/api/src/app/upload/usecases/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { MakeUploadEntry } from './make-upload-entry/make-upload-entry.usecase';
88
import { PaginateFileContent } from './paginate-file-content/paginate-file-content.usecase';
99
import { GetOriginalFileContent } from './get-original-file-content/get-original-file-content.usecase';
1010
import { GetUploadProcessInformation } from './get-upload-process-info/get-upload-process-info.usecase';
11+
import { UploadCleanupSchedulerService } from './uploadcleanup-scheduler/uploadcleanup-scheduler.service';
1112

1213
export const USE_CASES = [
1314
GetPreviewRows,
@@ -20,6 +21,7 @@ export const USE_CASES = [
2021
PaginateFileContent,
2122
GetOriginalFileContent,
2223
GetUploadProcessInformation,
24+
UploadCleanupSchedulerService,
2325
//
2426
];
2527

@@ -34,4 +36,5 @@ export {
3436
GetOriginalFileContent,
3537
GetUploadProcessInformation,
3638
PaginateFileContent,
39+
UploadCleanupSchedulerService,
3740
};

‎apps/api/src/app/upload/usecases/uploadcleanup-scheduler/uploadcleanup-scheduler.service.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import * as dayjs from 'dayjs';
2+
import { FileRepository, Upload } from '@impler/dal';
3+
import { Injectable } from '@nestjs/common';
4+
import { Cron } from '@nestjs/schedule';
5+
import { DalService } from '@impler/dal';
6+
import { StorageService } from '@impler/services';
7+
import { CRON_SCHEDULE } from '@shared/constants';
8+
9+
@Injectable()
10+
export class UploadCleanupSchedulerService {
11+
constructor(
12+
private readonly fileRepository: FileRepository,
13+
private readonly dalService: DalService,
14+
private storageService: StorageService
15+
) {}
16+
17+
@Cron(CRON_SCHEDULE.UPLOAD_CLEANUP_DEFAULT_CRON_TIME)
18+
async handleCleanupCronSchedular(cleanupDays: number = CRON_SCHEDULE.UPLOAD_CLEANUP_DAYS) {
19+
const cleanupDaysAgo = dayjs().subtract(cleanupDays, 'day').toDate();
20+
21+
const uploads = await Upload.find({
22+
uploadedDate: { $lt: cleanupDaysAgo },
23+
_uploadedFileId: { $exists: true, $ne: '' },
24+
});
25+
26+
if (uploads.length === 0) {
27+
return;
28+
}
29+
30+
for (const upload of uploads) {
31+
try {
32+
const files = await this.fileRepository.find({ _id: upload._uploadedFileId });
33+
await this.storageService.deleteFolder(upload.id);
34+
35+
await Promise.allSettled(
36+
files.map(async (file) => {
37+
try {
38+
await Upload.updateOne({ _uploadedFileId: file._id }, { $set: { _uploadedFileId: '' } });
39+
40+
// Delete file from storage and db
41+
try {
42+
await this.fileRepository.delete({ _id: file._id });
43+
} catch (error) {}
44+
45+
const collectionName = `${upload._id}-records`;
46+
try {
47+
const collections = await this.dalService.connection.db
48+
.listCollections({ name: collectionName })
49+
.toArray();
50+
51+
if (collections.length > 0) {
52+
const collection = this.dalService.connection.collection(collectionName);
53+
await collection.drop();
54+
}
55+
} catch (error) {}
56+
} catch (error) {}
57+
})
58+
);
59+
} catch (error) {}
60+
}
61+
}
62+
}

‎apps/api/src/config/env-validator.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ const validators: { [K in keyof any]: ValidatorSpec<any[K]> } = {
99
choices: [ENVTypesEnum.LOCAL, ENVTypesEnum.TEST, ENVTypesEnum.PROD, ENVTypesEnum.CI, ENVTypesEnum.LOCAL],
1010
default: ENVTypesEnum.LOCAL,
1111
}),
12+
1213
/*
1314
* S3_LOCAL_STACK: str(),
1415
* S3_BUCKET_NAME: str(),

‎apps/api/src/migrations/s3-to-azure-sample-file/s3-to-azure-sample-file.migration.ts‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/* eslint-disable multiline-comment-style */
2+
import '../../config';
3+
import { AppModule } from '../../app.module';
4+
5+
import { NestFactory } from '@nestjs/core';
6+
import { ExcelFileService } from '@shared/services';
7+
import { ColumnRepository, TemplateRepository } from '@impler/dal';
8+
import { AzureStorageService, FileNameService } from '@impler/services';
9+
import { ColumnTypesEnum, FileMimeTypesEnum, ColumnDelimiterEnum } from '@impler/shared';
10+
import { IExcelFileHeading } from '@shared/types/file.types';
11+
12+
export async function run() {
13+
let app;
14+
15+
try {
16+
console.log('start migration - regenerating sample excel files for all templates');
17+
18+
app = await NestFactory.create(AppModule, {
19+
logger: false,
20+
});
21+
22+
// Create services manually (since DI is not working for these services)
23+
const templateRepository = new TemplateRepository();
24+
const columnRepository = new ColumnRepository();
25+
const excelFileService = new ExcelFileService();
26+
const storageService = new AzureStorageService();
27+
const fileNameService = new FileNameService();
28+
29+
// Check if Azure connection string is available
30+
const azureConnectionString = process.env.AZURE_STORAGE_CONNECTION_STRING;
31+
let azureStorageService = null;
32+
33+
if (azureConnectionString) {
34+
try {
35+
azureStorageService = new AzureStorageService();
36+
} catch (error) {}
37+
}
38+
39+
const templates = await templateRepository.find(
40+
{
41+
sampleFileUrl: {
42+
$exists: true,
43+
$nin: ['', null],
44+
},
45+
},
46+
'sampleFileUrl name'
47+
);
48+
49+
if (templates.length === 0) {
50+
return;
51+
}
52+
53+
const templateIds = templates.map((template) => template._id);
54+
55+
const columns = await columnRepository.find({
56+
_templateId: {
57+
$in: templateIds,
58+
},
59+
});
60+
61+
for (let i = 0; i < templates.length; i++) {
62+
const template = templates[i];
63+
try {
64+
const templateColumns = columns.filter((column) => column._templateId.toString() === template._id.toString());
65+
66+
if (templateColumns.length > 0) {
67+
const columnKeys: IExcelFileHeading[] = templateColumns
68+
.map((columnItem) => ({
69+
key: columnItem.key,
70+
type: columnItem.type as ColumnTypesEnum,
71+
selectValues: columnItem.selectValues,
72+
isFrozen: columnItem.isFrozen,
73+
delimiter: columnItem.delimiter || ColumnDelimiterEnum.COMMA,
74+
isRequired: columnItem.isRequired,
75+
dateFormats: columnItem.dateFormats,
76+
allowMultiSelect: columnItem.allowMultiSelect,
77+
description: columnItem.description,
78+
}))
79+
.sort((a, b) => (a.isFrozen === b.isFrozen ? 0 : a.isFrozen ? -1 : 1));
80+
81+
const hasMultiSelect = templateColumns.some(
82+
(columnItem) => columnItem.type === ColumnTypesEnum.SELECT && columnItem.allowMultiSelect
83+
);
84+
85+
const fileName = fileNameService.getSampleFileName(template._id, hasMultiSelect);
86+
const sampleFileUrl = fileNameService.getSampleFileUrl(template._id, hasMultiSelect);
87+
88+
const sampleExcelFile = await excelFileService.getExcelFileForHeadings(columnKeys);
89+
90+
await storageService.uploadFile(fileName, sampleExcelFile, FileMimeTypesEnum.EXCELM);
91+
92+
await templateRepository.update({ _id: template._id }, { sampleFileUrl });
93+
94+
if (azureStorageService) {
95+
try {
96+
await azureStorageService.uploadFile(template.sampleFileUrl, sampleExcelFile, template._id);
97+
} catch (error) {}
98+
}
99+
}
100+
} catch (error) {
101+
continue;
102+
}
103+
}
104+
105+
console.log('end migration - regenerating sample excel files for all templates');
106+
} catch (error) {
107+
process.exit(1);
108+
} finally {
109+
if (app) {
110+
await app.close();
111+
}
112+
}
113+
}
114+
115+
run()
116+
.then(() => {
117+
console.log('Migration completed successfully');
118+
process.exit(0);
119+
})
120+
.catch((error) => {
121+
console.error('Migration failed:', error);
122+
process.exit(1);
123+
});

‎apps/queue-manager/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@impler/queue-manager",
3-
"version": "1.1.0",
3+
"version": "1.1.1",
44
"author": "implerhq",
55
"license": "MIT",
66
"private": true,

‎apps/web/components/import-feed/History.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ export function History() {
9595
key: 'download',
9696
width: 100,
9797
Cell(item) {
98-
return item.originalFileName ? (
98+
return item.originalFileName && item._uploadedFileId ? (
9999
<Tooltip label="Download original file" withArrow>
100100
<ActionIcon
101101
radius={0}

‎apps/web/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@impler/web",
3-
"version": "1.1.0",
3+
"version": "1.1.1",
44
"author": "implerhq",
55
"license": "MIT",
66
"private": true,

0 commit comments

Comments
 (0)