Skip to content

Commit 89224d0

Browse files
Arpit1119Arpit1119
committed
Fix: Fixed test migration Issues
1 parent 7c1bb55 commit 89224d0

File tree

3 files changed

+58
-42
lines changed

3 files changed

+58
-42
lines changed

‎api/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
"jsonwebtoken": "^9.0.2",
4646
"lowdb": "^7.0.1",
4747
"mkdirp": "^3.0.1",
48+
"p-limit": "^6.2.0",
4849
"path-to-regexp": "^8.2.0",
4950
"router": "^2.0.0",
5051
"shelljs": "^0.8.5",

‎api/src/services/contentful.service.ts‎

Lines changed: 35 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { v4 as uuidv4 } from "uuid";
55
import _ from "lodash";
66
import axios from "axios";
77
import jsonpath from "jsonpath";
8+
// import pLimit from 'p-limit';
89

910

1011
import {CHUNK_SIZE, MIGRATION_DATA_CONFIG } from "../constants/index.js";
@@ -24,6 +25,7 @@ const {
2425
RTE_REFERENCES_DIR_NAME,
2526
ENTRIES_DIR_NAME,
2627
ASSETS_DIR_NAME,
28+
GLOBAL_FIELDS_DIR_NAME,
2729
// FILE
2830
LOCALE_MASTER_LOCALE,
2931
LOCALE_FILE_NAME,
@@ -38,6 +40,7 @@ const {
3840
ENTRIES_MASTER_FILE,
3941
WEBHOOKS_FILE_NAME,
4042
RTE_REFERENCES_FILE_NAME,
43+
GLOBAL_FIELDS_FILE_NAME,
4144

4245
} = MIGRATION_DATA_CONFIG;
4346

@@ -152,8 +155,8 @@ const processField = (
152155
if (lang_value.sys) {
153156
const { linkType, id } = lang_value.sys;
154157

155-
if (linkType === "Entry" && id in entryId) return [entryId[id]];
156-
if (linkType === "Asset" && id in assetId) return assetId[id];
158+
if (linkType === "Entry" && id in entryId) return [entryId?.[id]];
159+
if (linkType === "Asset" && id in assetId) return assetId?.[id];
157160
}
158161

159162
// Handle arrays and nested objects
@@ -181,7 +184,7 @@ const processArrayFields = (array: any, entryId: any, assetId: any) => {
181184
if (id in entryId) {
182185
array.splice(i, 1, entryId[id]);
183186
} else if (id in assetId) {
184-
array.splice(i, 1, assetId[id]);
187+
array.splice(i, 1, assetId?.[id]);
185188
}
186189
});
187190
// Clean up empty objects
@@ -342,10 +345,10 @@ const saveAsset = async (
342345
await fs.promises.mkdir(assetPath, { recursive: true });
343346
// Write file as binary
344347
await fs.promises.writeFile(path.join(assetPath, fileName), Buffer.from(response.data), "binary");
345-
await customLogger(projectId, destination_stack_id, 'info', message);
346348
await writeFile(assetPath, `_contentstack_${assets.sys.id}.json`, assetData[assets.sys.id]);
347349
metadata.push({ uid: assets.sys.id, url: fileUrl, filename: fileName });
348350
delete failedJSON[assets.sys.id];
351+
await customLogger(projectId, destination_stack_id, 'info', message);
349352
} catch (err: any) {
350353
if (retryCount === 1) {
351354
failedJSON[assets.sys.id] = {
@@ -401,31 +404,41 @@ const createAssets = async (packagePath: any, destination_stack_id:string, proje
401404
const failedJSON: any = {};
402405
const assetData: any = {};
403406
const metadata: AssetMetaData[] = [];
404-
407+
const fileMeta = { "1": ASSETS_SCHEMA_FILE };
405408
const assets = JSON.parse(data)?.assets;
406409

407410
if (assets && assets.length > 0) {
408411
const tasks = assets.map(
409-
async (asset: any) =>
410-
await saveAsset(asset, failedJSON, assetData, metadata, projectId, destination_stack_id, 0)
412+
async (asset: any, index: any) =>
413+
saveAsset(asset, failedJSON, assetData, metadata, projectId, destination_stack_id, 0)
411414
);
415+
416+
// This code is intentionally commented out
417+
// for testing purposes.
418+
419+
// const limit = pLimit(10); // Limit concurrent operations to 10
420+
// const tasks = assets.map((asset: any) =>
421+
// limit(() => saveAsset(asset, failedJSON, assetData, metadata, projectId, destination_stack_id, 0))
422+
// );
423+
// await Promise.all(tasks);
424+
412425
await Promise.all(tasks);
413426
const assetMasterFolderPath = path.join(assetsSave, ASSETS_FAILED_FILE);
414427

415428
await writeOneFile(path.join(assetsSave, ASSETS_SCHEMA_FILE), assetData);
416-
const chunks: { [key: string]: any } = makeChunks(assetData);
417-
const refs: any = {};
418-
419-
for (const [index, chunkId] of Object.keys(chunks).entries()) {
420-
refs[index + 1] = `${chunkId}-${ASSETS_FILE_NAME}`;
421-
await writeOneFile(
422-
path.join(assetsSave, `${chunkId}-${ASSETS_FILE_NAME}`),
423-
chunks[chunkId]
424-
);
425-
}
426-
427-
await writeOneFile(path.join(assetsSave, ASSETS_FILE_NAME), refs);
428-
await writeOneFile(path.join(assetsSave, ASSETS_METADATA_FILE), metadata);
429+
// const chunks: { [key: string]: any } = makeChunks(assetData);
430+
// const refs: any = {};
431+
432+
// for (const [index, chunkId] of Object.keys(chunks).entries()) {
433+
// refs[index + 1] = `${chunkId}-${ASSETS_FILE_NAME}`;
434+
// await writeOneFile(
435+
// path.join(assetsSave, `${chunkId}-${ASSETS_FILE_NAME}`),
436+
// chunks[chunkId]
437+
// );
438+
// }
439+
440+
await writeOneFile(path.join(assetsSave, ASSETS_FILE_NAME), fileMeta);
441+
// await writeOneFile(path.join(assetsSave, ASSETS_METADATA_FILE), metadata);
429442
failedJSON && await writeFile(assetMasterFolderPath, ASSETS_FAILED_FILE, failedJSON);
430443
} else {
431444
const message = getLogMessage(
@@ -720,6 +733,7 @@ const createEntry = async (packagePath: any, destination_stack_id:string, projec
720733
const createLocale = async (packagePath: string, destination_stack_id:string, projectId:string) => {
721734
const srcFunc = 'createLocale';
722735
const localeSave = path.join(DATA, destination_stack_id, LOCALE_DIR_NAME);
736+
const globalFieldSave = path.join(DATA, destination_stack_id, GLOBAL_FIELDS_DIR_NAME);
723737

724738
try {
725739
const msLocale: Record<string, Locale> = {};
@@ -773,6 +787,7 @@ const createLocale = async (packagePath: string, destination_stack_id:string, pr
773787
await writeFile(localeSave, LOCALE_FILE_NAME, allLocales)
774788
await writeFile(localeSave, LOCALE_MASTER_LOCALE, msLocale)
775789
await writeFile(localeSave, LOCALE_CF_LANGUAGE, localeList)
790+
await writeFile(globalFieldSave, GLOBAL_FIELDS_FILE_NAME, [])
776791
const message = getLogMessage(
777792
srcFunc,
778793
`locales have been successfully transformed.`,

‎api/src/services/contentful/jsonRTE.ts‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -85,15 +85,15 @@ function parseDocument(obj: any, lang?: LangType, destination_stack_id?:StackId)
8585
};
8686
}
8787

88-
function parseTable(obj: any): any {
88+
function parseTable(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
8989
const rowCount = obj.content.length;
9090
const colCount = Math.max(...obj.content.map((e: any) => e.content.length));
9191
const attrs = {
9292
rows: rowCount,
9393
cols: colCount,
9494
colWidths: Array(colCount).fill(250),
9595
};
96-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).concat(parsers.get('tbody')?.(obj)).filter(Boolean);
96+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).concat(parsers.get('tbody')?.(obj)).filter(Boolean);
9797

9898
return {
9999
type: 'table',
@@ -103,19 +103,19 @@ function parseTable(obj: any): any {
103103
};
104104
}
105105

106-
function parseTableRow(obj: any): any {
106+
function parseTableRow(obj: any,lang?: LangType, destination_stack_id?:StackId): any {
107107
const types = new Set<string>();
108108
const children = obj.content.map((e: any) => {
109109
types.add(e.nodeType);
110-
return parsers.get(e.nodeType)?.(e);
110+
return parsers.get(e.nodeType)?.(e, lang, destination_stack_id);
111111
}).filter(Boolean);
112112

113113
const type = types.has('table-header-cell') ? 'thead' : '';
114114
return children.length ? { type, attrs: {}, uid: generateUID('tabletype'), children } : null;
115115
}
116116

117-
function parseHeadTR(obj: any[]): any {
118-
const children = obj.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
117+
function parseHeadTR(obj: any[],lang?: LangType, destination_stack_id?:StackId): any {
118+
const children = obj.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean);
119119
return {
120120
type: 'tr',
121121
attrs: {},
@@ -124,8 +124,8 @@ function parseHeadTR(obj: any[]): any {
124124
};
125125
}
126126

127-
function parseTableHead(obj: any): any {
128-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
127+
function parseTableHead(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
128+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean);
129129
return {
130130
type: 'th',
131131
attrs: {},
@@ -134,8 +134,8 @@ function parseTableHead(obj: any): any {
134134
};
135135
}
136136

137-
function parseTBody(obj: any): any {
138-
const children = obj.content.map((e: any) => parsers.get('body-tr')?.(e)).filter(Boolean);
137+
function parseTBody(obj: any,lang?: LangType, destination_stack_id?:StackId ): any {
138+
const children = obj.content.map((e: any) => parsers.get('body-tr')?.(e, lang, destination_stack_id)).filter(Boolean);
139139
return {
140140
type: 'tbody',
141141
attrs: {},
@@ -144,18 +144,18 @@ function parseTBody(obj: any): any {
144144
};
145145
}
146146

147-
function parseBodyTR(obj: any): any {
148-
const children = obj.content.filter((e: any) => e.nodeType === 'table-cell').map((e: any) => parsers.get('table-cell')?.(e)).filter(Boolean);
147+
function parseBodyTR(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
148+
const children = obj.content.filter((e: any) => e.nodeType === 'table-cell').map((e: any) => parsers.get('table-cell')?.(e, lang, destination_stack_id)).filter(Boolean);
149149
return children.length ? { type: 'tr', attrs: {}, uid: generateUID('tr'), children } : null;
150150
}
151151

152-
function parseTableBody(obj: any): any {
153-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
152+
function parseTableBody(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
153+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean);
154154
return children.length ? { type: 'td', attrs: {}, uid: generateUID('td'), children } : null;
155155
}
156156

157-
function parseParagraph(obj: any, lang?: LangType): any {
158-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang)).filter(Boolean);
157+
function parseParagraph(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
158+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean);
159159
return {
160160
type: 'p',
161161
attrs: {},
@@ -181,8 +181,8 @@ function parseHR(): any {
181181
};
182182
}
183183

184-
function parseUL(obj: any): any {
185-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
184+
function parseUL(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
185+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e,lang, destination_stack_id)).filter(Boolean);
186186
return {
187187
type: 'ul',
188188
attrs: {},
@@ -192,8 +192,8 @@ function parseUL(obj: any): any {
192192
};
193193
}
194194

195-
function parseOL(obj: any): any {
196-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).filter(Boolean);
195+
function parseOL(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
196+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).filter(Boolean);
197197
return {
198198
type: 'ol',
199199
attrs: {},
@@ -203,8 +203,8 @@ function parseOL(obj: any): any {
203203
};
204204
}
205205

206-
function parseLI(obj: any): any {
207-
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e)).flat().filter(Boolean);
206+
function parseLI(obj: any, lang?: LangType, destination_stack_id?:StackId): any {
207+
const children = obj.content.map((e: any) => parsers.get(e.nodeType)?.(e, lang, destination_stack_id)).flat().filter(Boolean);
208208
return {
209209
type: 'li',
210210
attrs: {},

0 commit comments

Comments
 (0)